2024-06-11T13:26:46.454 INFO:root:teuthology version: 0.0.1.dev303+g861a8dc 2024-06-11T13:26:46.454 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --name teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326 --verbose -- /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/orig.config.yaml 2024-06-11T13:26:46.492 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-11T13:26:46.570 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7749326' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 3f51ea3944d39ca13aa40153c9a4ee298306679d ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3f51ea3944d39ca13aa40153c9a4ee298306679d selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 3f51ea3944d39ca13aa40153c9a4ee298306679d owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 6336 sha1: 3f51ea3944d39ca13aa40153c9a4ee298306679d sleep_before_teardown: 0 subset: 5/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_3f51ea3944d39ca13aa40153c9a4ee298306679d/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 3f51ea3944d39ca13aa40153c9a4ee298306679d targets: smithi069.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBITYW4diWnfTvSHyoabkqWfcL82NKgFXGQoM6tQDSJAXebNnxq3bM4hm2fc8r2fYpUVpZgkbscUIKEjCJgwTCzw= smithi135.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSjcPPvUBKV8A2f1xWUXVuAoQzznomkEwHVQriZfCk1KO3KFQuVP0g/DPCULt3qvhSjJeb5VOm7dy0AW0JEL50= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 861a8dcf7aa816a26e13f039336f7ed0a9aec0fa timestamp: 2024-06-10_21:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-06-11T13:26:46.570 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_3f51ea3944d39ca13aa40153c9a4ee298306679d/qa; will attempt to use it 2024-06-11T13:26:46.571 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_3f51ea3944d39ca13aa40153c9a4ee298306679d/qa/tasks 2024-06-11T13:26:46.571 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-06-11T13:26:46.573 INFO:teuthology.task.internal:Checking packages... 2024-06-11T13:26:46.593 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3f51ea3944d39ca13aa40153c9a4ee298306679d' 2024-06-11T13:26:46.594 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-06-11T13:26:46.594 INFO:teuthology.packaging:ref: None 2024-06-11T13:26:46.594 INFO:teuthology.packaging:tag: None 2024-06-11T13:26:46.594 INFO:teuthology.packaging:branch: squid 2024-06-11T13:26:46.594 INFO:teuthology.packaging:sha1: 3f51ea3944d39ca13aa40153c9a4ee298306679d 2024-06-11T13:26:46.594 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-06-11T13:26:46.773 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2976.g3f51ea39 2024-06-11T13:26:46.775 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-06-11T13:26:46.781 INFO:teuthology.task.internal:no buildpackages task found 2024-06-11T13:26:46.781 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-06-11T13:26:46.866 INFO:teuthology.task.internal:Saving configuration 2024-06-11T13:26:46.880 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-06-11T13:26:46.915 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-06-11T13:26:46.938 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-11 13:21:55.711273', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBITYW4diWnfTvSHyoabkqWfcL82NKgFXGQoM6tQDSJAXebNnxq3bM4hm2fc8r2fYpUVpZgkbscUIKEjCJgwTCzw='} 2024-06-11T13:26:46.956 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi135.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-06-11 13:21:55.712550', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNSjcPPvUBKV8A2f1xWUXVuAoQzznomkEwHVQriZfCk1KO3KFQuVP0g/DPCULt3qvhSjJeb5VOm7dy0AW0JEL50='} 2024-06-11T13:26:46.956 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-06-11T13:26:47.006 INFO:teuthology.task.internal:roles: ubuntu@smithi069.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-06-11T13:26:47.006 INFO:teuthology.task.internal:roles: ubuntu@smithi135.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-06-11T13:26:47.006 INFO:teuthology.run_tasks:Running task console_log... 2024-06-11T13:26:47.128 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fdcfefafac0>, signals=[15]) 2024-06-11T13:26:47.128 INFO:teuthology.run_tasks:Running task internal.connect... 2024-06-11T13:26:47.149 INFO:teuthology.task.internal:Opening connections... 2024-06-11T13:26:47.150 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:26:47.151 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:26:47.226 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi135.front.sepia.ceph.com 2024-06-11T13:26:47.227 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:26:47.300 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-06-11T13:26:47.308 DEBUG:teuthology.orchestra.run.smithi069:> uname -m 2024-06-11T13:26:47.326 INFO:teuthology.orchestra.run.smithi069.stdout:x86_64 2024-06-11T13:26:47.326 DEBUG:teuthology.orchestra.run.smithi069:> cat /etc/os-release 2024-06-11T13:26:47.383 INFO:teuthology.orchestra.run.smithi069.stdout:NAME="CentOS Stream" 2024-06-11T13:26:47.383 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION="9" 2024-06-11T13:26:47.383 INFO:teuthology.orchestra.run.smithi069.stdout:ID="centos" 2024-06-11T13:26:47.383 INFO:teuthology.orchestra.run.smithi069.stdout:ID_LIKE="rhel fedora" 2024-06-11T13:26:47.383 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION_ID="9" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:PLATFORM_ID="platform:el9" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:ANSI_COLOR="0;31" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:LOGO="fedora-logo-icon" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:HOME_URL="https://centos.org/" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-11T13:26:47.384 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-11T13:26:47.385 INFO:teuthology.lock.ops:Updating smithi069.front.sepia.ceph.com on lock server 2024-06-11T13:26:47.410 DEBUG:teuthology.orchestra.run.smithi135:> uname -m 2024-06-11T13:26:47.428 INFO:teuthology.orchestra.run.smithi135.stdout:x86_64 2024-06-11T13:26:47.428 DEBUG:teuthology.orchestra.run.smithi135:> cat /etc/os-release 2024-06-11T13:26:47.485 INFO:teuthology.orchestra.run.smithi135.stdout:NAME="CentOS Stream" 2024-06-11T13:26:47.485 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION="9" 2024-06-11T13:26:47.485 INFO:teuthology.orchestra.run.smithi135.stdout:ID="centos" 2024-06-11T13:26:47.485 INFO:teuthology.orchestra.run.smithi135.stdout:ID_LIKE="rhel fedora" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:VERSION_ID="9" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:PLATFORM_ID="platform:el9" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:PRETTY_NAME="CentOS Stream 9" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:ANSI_COLOR="0;31" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:LOGO="fedora-logo-icon" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:HOME_URL="https://centos.org/" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-06-11T13:26:47.486 INFO:teuthology.orchestra.run.smithi135.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-06-11T13:26:47.486 INFO:teuthology.lock.ops:Updating smithi135.front.sepia.ceph.com on lock server 2024-06-11T13:26:47.501 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-06-11T13:26:47.510 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-06-11T13:26:47.575 INFO:teuthology.task.internal:Checking for old test directory... 2024-06-11T13:26:47.575 DEBUG:teuthology.orchestra.run.smithi069:> test '!' -e /home/ubuntu/cephtest 2024-06-11T13:26:47.578 DEBUG:teuthology.orchestra.run.smithi135:> test '!' -e /home/ubuntu/cephtest 2024-06-11T13:26:47.594 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-06-11T13:26:47.639 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-06-11T13:26:47.639 DEBUG:teuthology.orchestra.run.smithi069:> test -z $(ls -A /var/lib/ceph) 2024-06-11T13:26:47.641 DEBUG:teuthology.orchestra.run.smithi135:> test -z $(ls -A /var/lib/ceph) 2024-06-11T13:26:47.664 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-06-11T13:26:47.768 INFO:teuthology.run_tasks:Running task kernel... 2024-06-11T13:26:47.815 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-06-11T13:26:47.816 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-06-11T13:26:47.816 DEBUG:teuthology.orchestra.run.smithi069:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-11T13:26:47.816 DEBUG:teuthology.orchestra.run.smithi135:> test -f /run/.containerenv -o -f /.dockerenv 2024-06-11T13:26:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T13:26:47.833 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2024-06-11T13:26:47.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T13:26:47.834 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2024-06-11T13:26:47.889 INFO:teuthology.orchestra.run.smithi069.stdout:5.14.0-437.el9.x86_64 2024-06-11T13:26:47.890 INFO:teuthology.task.kernel:Running kernel on smithi069: 5.14.0-437.el9.x86_64 2024-06-11T13:26:47.890 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum install -y kernel 2024-06-11T13:26:47.891 INFO:teuthology.orchestra.run.smithi135.stdout:5.14.0-437.el9.x86_64 2024-06-11T13:26:47.891 INFO:teuthology.task.kernel:Running kernel on smithi135: 5.14.0-437.el9.x86_64 2024-06-11T13:26:47.891 DEBUG:teuthology.orchestra.run.smithi135:> sudo yum install -y kernel 2024-06-11T13:26:51.522 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - BaseOS 4.1 MB/s | 8.1 MB 00:01 2024-06-11T13:26:52.736 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - BaseOS 2.5 MB/s | 8.1 MB 00:03 2024-06-11T13:26:53.407 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-06-11T13:26:54.646 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-06-11T13:26:58.632 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.2 MB 00:01 2024-06-11T13:27:00.189 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - CRB 3.1 MB/s | 6.2 MB 00:01 2024-06-11T13:27:00.725 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - Extras packages 22 kB/s | 17 kB 00:00 2024-06-11T13:27:01.543 INFO:teuthology.orchestra.run.smithi069.stdout:Extra Packages for Enterprise Linux 31 MB/s | 21 MB 00:00 2024-06-11T13:27:01.714 INFO:teuthology.orchestra.run.smithi135.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 17 kB 00:00 2024-06-11T13:27:02.876 INFO:teuthology.orchestra.run.smithi135.stdout:Extra Packages for Enterprise Linux 21 MB/s | 21 MB 00:01 2024-06-11T13:27:06.815 INFO:teuthology.orchestra.run.smithi069.stdout:lab-extras 19 kB/s | 1.7 kB 00:00 2024-06-11T13:27:08.006 INFO:teuthology.orchestra.run.smithi135.stdout:lab-extras 35 kB/s | 1.7 kB 00:00 2024-06-11T13:27:08.830 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-11T13:27:08.830 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-11T13:27:08.925 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout:Installing: 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 5.14.0-452.el9 baseos 893 k 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout:Installing dependencies: 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-core x86_64 5.14.0-452.el9 baseos 16 M 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules x86_64 5.14.0-452.el9 baseos 35 M 2024-06-11T13:27:08.937 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules-core x86_64 5.14.0-452.el9 baseos 29 M 2024-06-11T13:27:08.938 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:27:08.938 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2024-06-11T13:27:08.938 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-06-11T13:27:08.938 INFO:teuthology.orchestra.run.smithi069.stdout:Install 4 Packages 2024-06-11T13:27:08.938 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:27:08.938 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 80 M 2024-06-11T13:27:08.939 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 124 M 2024-06-11T13:27:08.939 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2024-06-11T13:27:09.290 INFO:teuthology.orchestra.run.smithi069.stdout:[MIRROR] kernel-5.14.0-452.el9.x86_64.rpm: Status code: 404 for https://download-cc-rdu01.fedoraproject.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-452.el9.x86_64.rpm (IP: 8.43.85.72) 2024-06-11T13:27:09.291 INFO:teuthology.orchestra.run.smithi069.stdout:[MIRROR] kernel-core-5.14.0-452.el9.x86_64.rpm: Status code: 404 for https://download-cc-rdu01.fedoraproject.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-452.el9.x86_64.rpm (IP: 8.43.85.72) 2024-06-11T13:27:09.295 INFO:teuthology.orchestra.run.smithi069.stdout:[MIRROR] kernel-modules-5.14.0-452.el9.x86_64.rpm: Status code: 404 for https://download-cc-rdu01.fedoraproject.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-452.el9.x86_64.rpm (IP: 8.43.85.72) 2024-06-11T13:27:09.475 INFO:teuthology.orchestra.run.smithi069.stdout:(1/4): kernel-5.14.0-452.el9.x86_64.rpm 3.3 MB/s | 893 kB 00:00 2024-06-11T13:27:09.988 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-06-11T13:27:09.989 INFO:teuthology.orchestra.run.smithi135.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-06-11T13:27:10.084 INFO:teuthology.orchestra.run.smithi135.stdout:Dependencies resolved. 2024-06-11T13:27:10.096 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout: Package Architecture Version Repository Size 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout:Installing: 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout: kernel x86_64 5.14.0-452.el9 baseos 893 k 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout:Installing dependencies: 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-core x86_64 5.14.0-452.el9 baseos 16 M 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules x86_64 5.14.0-452.el9 baseos 35 M 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-core x86_64 5.14.0-452.el9 baseos 29 M 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction Summary 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout:================================================================================ 2024-06-11T13:27:10.097 INFO:teuthology.orchestra.run.smithi135.stdout:Install 4 Packages 2024-06-11T13:27:10.098 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:27:10.098 INFO:teuthology.orchestra.run.smithi135.stdout:Total download size: 80 M 2024-06-11T13:27:10.098 INFO:teuthology.orchestra.run.smithi135.stdout:Installed size: 124 M 2024-06-11T13:27:10.098 INFO:teuthology.orchestra.run.smithi135.stdout:Downloading Packages: 2024-06-11T13:27:10.446 INFO:teuthology.orchestra.run.smithi135.stdout:[MIRROR] kernel-5.14.0-452.el9.x86_64.rpm: Status code: 404 for https://download-cc-rdu01.fedoraproject.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-452.el9.x86_64.rpm (IP: 8.43.85.72) 2024-06-11T13:27:10.446 INFO:teuthology.orchestra.run.smithi135.stdout:[MIRROR] kernel-core-5.14.0-452.el9.x86_64.rpm: Status code: 404 for https://download-cc-rdu01.fedoraproject.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-452.el9.x86_64.rpm (IP: 8.43.85.72) 2024-06-11T13:27:10.451 INFO:teuthology.orchestra.run.smithi135.stdout:[MIRROR] kernel-modules-5.14.0-452.el9.x86_64.rpm: Status code: 404 for https://download-cc-rdu01.fedoraproject.org/pub/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-452.el9.x86_64.rpm (IP: 8.43.85.72) 2024-06-11T13:27:10.783 INFO:teuthology.orchestra.run.smithi135.stdout:(1/4): kernel-5.14.0-452.el9.x86_64.rpm 2.1 MB/s | 893 kB 00:00 2024-06-11T13:27:14.533 INFO:teuthology.orchestra.run.smithi135.stdout:(2/4): kernel-core-5.14.0-452.el9.x86_64.rpm 3.8 MB/s | 16 MB 00:04 2024-06-11T13:27:14.808 INFO:teuthology.orchestra.run.smithi069.stdout:(2/4): kernel-core-5.14.0-452.el9.x86_64.rpm 2.8 MB/s | 16 MB 00:05 2024-06-11T13:27:17.225 INFO:teuthology.orchestra.run.smithi135.stdout:(3/4): kernel-modules-core-5.14.0-452.el9.x86_6 4.5 MB/s | 29 MB 00:06 2024-06-11T13:27:18.550 INFO:teuthology.orchestra.run.smithi135.stdout:(4/4): kernel-modules-5.14.0-452.el9.x86_64.rpm 4.2 MB/s | 35 MB 00:08 2024-06-11T13:27:18.553 INFO:teuthology.orchestra.run.smithi135.stdout:-------------------------------------------------------------------------------- 2024-06-11T13:27:18.553 INFO:teuthology.orchestra.run.smithi135.stdout:Total 9.5 MB/s | 80 MB 00:08 2024-06-11T13:27:19.041 INFO:teuthology.orchestra.run.smithi069.stdout:(3/4): kernel-modules-core-5.14.0-452.el9.x86_6 3.0 MB/s | 29 MB 00:09 2024-06-11T13:27:19.247 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction check 2024-06-11T13:27:19.584 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction check succeeded. 2024-06-11T13:27:19.584 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction test 2024-06-11T13:27:20.058 INFO:teuthology.orchestra.run.smithi069.stdout:(4/4): kernel-modules-5.14.0-452.el9.x86_64.rpm 3.2 MB/s | 35 MB 00:10 2024-06-11T13:27:20.060 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2024-06-11T13:27:20.061 INFO:teuthology.orchestra.run.smithi069.stdout:Total 7.2 MB/s | 80 MB 00:11 2024-06-11T13:27:20.563 INFO:teuthology.orchestra.run.smithi135.stdout:Transaction test succeeded. 2024-06-11T13:27:20.563 INFO:teuthology.orchestra.run.smithi135.stdout:Running transaction 2024-06-11T13:27:20.757 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2024-06-11T13:27:21.096 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2024-06-11T13:27:21.096 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2024-06-11T13:27:22.062 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2024-06-11T13:27:22.063 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2024-06-11T13:27:22.958 INFO:teuthology.orchestra.run.smithi135.stdout: Preparing : 1/1 2024-06-11T13:27:23.576 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-modules-core-5.14.0-452.el9.x86_64 1/4 2024-06-11T13:27:23.664 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-11T13:27:24.469 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2024-06-11T13:27:25.094 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-modules-core-5.14.0-452.el9.x86_64 1/4 2024-06-11T13:27:25.174 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-11T13:27:26.289 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-11T13:27:26.452 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-11T13:27:27.820 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-11T13:27:27.976 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-11T13:27:31.165 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-11T13:27:31.217 INFO:teuthology.orchestra.run.smithi135.stdout: Installing : kernel-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:27:32.710 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-11T13:27:32.846 INFO:teuthology.orchestra.run.smithi069.stdout: Installing : kernel-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:27:35.943 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:27:37.588 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:05.413 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:05.452 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:05.799 INFO:teuthology.orchestra.run.smithi135.stdout: Running scriptlet: kernel-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:05.799 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-5.14.0-452.el9.x86_64 1/4 2024-06-11T13:28:05.799 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-11T13:28:05.799 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: Verifying : kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout:Installed: 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-5.14.0-452.el9.x86_64 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-core-5.14.0-452.el9.x86_64 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-5.14.0-452.el9.x86_64 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: kernel-modules-core-5.14.0-452.el9.x86_64 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:28:06.639 INFO:teuthology.orchestra.run.smithi135.stdout:Complete! 2024-06-11T13:28:06.899 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-core-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:06.939 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-modules-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:07.004 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2024-06-11T13:28:07.086 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-11T13:28:07.087 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-11T13:28:07.087 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-452.el9.x86_64 2024-06-11T13:28:07.087 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-11T13:28:07.087 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-11T13:28:07.087 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-06-11T13:28:07.088 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi135.front.sepia.ceph.com, path=None, version=distro) 2024-06-11T13:28:07.088 DEBUG:teuthology.orchestra.run.smithi135:> rpm -q kernel | sort -rV | head -n 1 2024-06-11T13:28:07.140 INFO:teuthology.orchestra.run.smithi135.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-11T13:28:07.140 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-11T13:28:07.140 DEBUG:teuthology.orchestra.run.smithi135:> sudo rpm -qi grub2-tools 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Name : grub2-tools 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Epoch : 1 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Version : 2.06 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Release : 68.el9 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Architecture: x86_64 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Group : Unspecified 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Size : 8264254 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:License : GPLv3+ 2024-06-11T13:28:07.171 INFO:teuthology.orchestra.run.smithi135.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Packager : builder@centos.org 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Vendor : CentOS 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Summary : Support tools for GRUB. 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:Description : 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout:hardware devices. 2024-06-11T13:28:07.172 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:28:07.173 INFO:teuthology.orchestra.run.smithi135.stdout:This subpackage provides tools for support of all platforms. 2024-06-11T13:28:07.173 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-11T13:28:07.173 INFO:teuthology.task.kernel:Updating grub on smithi135 to boot 5.14.0-452.el9.x86_64 2024-06-11T13:28:07.173 DEBUG:teuthology.orchestra.run.smithi135:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-11T13:28:07.309 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:07.309 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-5.14.0-452.el9.x86_64 1/4 2024-06-11T13:28:07.309 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-core-5.14.0-452.el9.x86_64 2/4 2024-06-11T13:28:07.309 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-modules-5.14.0-452.el9.x86_64 3/4 2024-06-11T13:28:07.843 INFO:teuthology.orchestra.run.smithi135.stderr:Generating grub configuration file ... 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-modules-core-5.14.0-452.el9.x86_64 4/4 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout:Installed: 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-5.14.0-452.el9.x86_64 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-core-5.14.0-452.el9.x86_64 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules-5.14.0-452.el9.x86_64 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-modules-core-5.14.0-452.el9.x86_64 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:28:08.123 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2024-06-11T13:28:08.480 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2024-06-11T13:28:08.553 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-11T13:28:08.554 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-11T13:28:08.554 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-452.el9.x86_64 2024-06-11T13:28:08.554 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-06-11T13:28:08.554 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-06-11T13:28:08.554 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-06-11T13:28:08.554 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi069.front.sepia.ceph.com, path=None, version=distro) 2024-06-11T13:28:08.555 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2024-06-11T13:28:08.607 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-5.14.0-452.el9.x86_64 2024-06-11T13:28:08.607 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-452.el9.x86_64 2024-06-11T13:28:08.607 DEBUG:teuthology.orchestra.run.smithi069:> sudo rpm -qi grub2-tools 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Name : grub2-tools 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Epoch : 1 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Version : 2.06 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Release : 68.el9 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Architecture: x86_64 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Group : Unspecified 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:Size : 8264254 2024-06-11T13:28:08.639 INFO:teuthology.orchestra.run.smithi069.stdout:License : GPLv3+ 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Packager : builder@centos.org 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Vendor : CentOS 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:URL : http://www.gnu.org/software/grub/ 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Summary : Support tools for GRUB. 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:Description : 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:28:08.640 INFO:teuthology.orchestra.run.smithi069.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-06-11T13:28:08.641 INFO:teuthology.orchestra.run.smithi069.stdout:customizable bootloader with modular architecture. It supports a rich 2024-06-11T13:28:08.641 INFO:teuthology.orchestra.run.smithi069.stdout:variety of kernel formats, file systems, computer architectures and 2024-06-11T13:28:08.641 INFO:teuthology.orchestra.run.smithi069.stdout:hardware devices. 2024-06-11T13:28:08.641 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:28:08.641 INFO:teuthology.orchestra.run.smithi069.stdout:This subpackage provides tools for support of all platforms. 2024-06-11T13:28:08.642 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-06-11T13:28:08.642 INFO:teuthology.task.kernel:Updating grub on smithi069 to boot 5.14.0-452.el9.x86_64 2024-06-11T13:28:08.642 DEBUG:teuthology.orchestra.run.smithi069:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-06-11T13:28:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:Generating grub configuration file ... 2024-06-11T13:28:09.286 INFO:teuthology.orchestra.run.smithi135.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-11T13:28:09.321 INFO:teuthology.orchestra.run.smithi135.stderr:done 2024-06-11T13:28:09.323 DEBUG:teuthology.orchestra.run.smithi135:> mktemp 2024-06-11T13:28:09.379 INFO:teuthology.orchestra.run.smithi135.stdout:/tmp/tmp.KTu5nF6iRc 2024-06-11T13:28:09.379 DEBUG:teuthology.orchestra.run.smithi135:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.KTu5nF6iRc 2024-06-11T13:28:09.445 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 0666 /tmp/tmp.KTu5nF6iRc 2024-06-11T13:28:09.595 DEBUG:teuthology.orchestra.remote:smithi135:/tmp/tmp.KTu5nF6iRc is 6KB 2024-06-11T13:28:09.644 DEBUG:teuthology.orchestra.run.smithi135:> rm -fr /tmp/tmp.KTu5nF6iRc 2024-06-11T13:28:09.658 DEBUG:teuthology.orchestra.run.smithi135:> sudo /bin/ls /boot/loader/entries || true 2024-06-11T13:28:09.725 INFO:teuthology.orchestra.run.smithi135.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-11T13:28:09.725 INFO:teuthology.orchestra.run.smithi135.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-11T13:28:09.725 INFO:teuthology.orchestra.run.smithi135.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-11T13:28:09.725 INFO:teuthology.orchestra.run.smithi135.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64.conf 2024-06-11T13:28:09.726 DEBUG:teuthology.orchestra.run.smithi135:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64 2024-06-11T13:28:09.877 DEBUG:teuthology.orchestra.run.smithi135:> sudo shutdown -r now 2024-06-11T13:28:10.630 INFO:teuthology.orchestra.run.smithi069.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-06-11T13:28:10.677 INFO:teuthology.orchestra.run.smithi069.stderr:done 2024-06-11T13:28:10.680 DEBUG:teuthology.orchestra.run.smithi069:> mktemp 2024-06-11T13:28:10.711 INFO:teuthology.orchestra.run.smithi069.stdout:/tmp/tmp.dX2hKDq8Es 2024-06-11T13:28:10.711 DEBUG:teuthology.orchestra.run.smithi069:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.dX2hKDq8Es 2024-06-11T13:28:10.778 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0666 /tmp/tmp.dX2hKDq8Es 2024-06-11T13:28:10.915 DEBUG:teuthology.orchestra.remote:smithi069:/tmp/tmp.dX2hKDq8Es is 6KB 2024-06-11T13:28:10.963 DEBUG:teuthology.orchestra.run.smithi069:> rm -fr /tmp/tmp.dX2hKDq8Es 2024-06-11T13:28:10.978 DEBUG:teuthology.orchestra.run.smithi069:> sudo /bin/ls /boot/loader/entries || true 2024-06-11T13:28:11.049 INFO:teuthology.orchestra.run.smithi069.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-06-11T13:28:11.049 INFO:teuthology.orchestra.run.smithi069.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-06-11T13:28:11.049 INFO:teuthology.orchestra.run.smithi069.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-06-11T13:28:11.049 INFO:teuthology.orchestra.run.smithi069.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64.conf 2024-06-11T13:28:11.050 DEBUG:teuthology.orchestra.run.smithi069:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-452.el9.x86_64 2024-06-11T13:28:11.182 DEBUG:teuthology.orchestra.run.smithi069:> sudo shutdown -r now 2024-06-11T13:28:39.906 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-06-11T13:28:39.907 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-06-11T13:28:39.908 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:28:41.186 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-06-11T13:28:41.186 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-06-11T13:28:41.187 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:29:14.527 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2024-06-11T13:29:23.534 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-06-11T13:29:23.535 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:29:41.206 DEBUG:teuthology.orchestra.remote:timed out 2024-06-11T13:29:41.918 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.135 2024-06-11T13:29:50.211 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-06-11T13:29:50.212 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:29:53.577 DEBUG:teuthology.orchestra.run.smithi069:> true 2024-06-11T13:29:53.920 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-06-11T13:29:53.921 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:29:53.967 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-06-11T13:29:53.967 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-452.el9.x86_64"... 2024-06-11T13:29:53.968 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2024-06-11T13:29:53.982 INFO:teuthology.orchestra.run.smithi069.stdout:5.14.0-452.el9.x86_64 2024-06-11T13:29:53.982 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-452.el9.x86_64 vs 5.14.0-452.el9.x86_64 2024-06-11T13:29:53.982 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-11T13:29:53.982 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-11T13:29:54.690 DEBUG:teuthology.orchestra.run.smithi135:> true 2024-06-11T13:29:54.983 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-06-11T13:29:54.983 DEBUG:teuthology.orchestra.run.smithi069:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-11T13:29:55.052 INFO:teuthology.orchestra.run.smithi069.stdout:ttyS1 2024-06-11T13:29:55.089 DEBUG:teuthology.parallel:result is None 2024-06-11T13:29:55.142 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-06-11T13:29:55.143 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-452.el9.x86_64"... 2024-06-11T13:29:55.143 DEBUG:teuthology.orchestra.run.smithi135:> uname -r 2024-06-11T13:29:55.157 INFO:teuthology.orchestra.run.smithi135.stdout:5.14.0-452.el9.x86_64 2024-06-11T13:29:55.157 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-452.el9.x86_64 vs 5.14.0-452.el9.x86_64 2024-06-11T13:29:55.157 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-06-11T13:29:55.157 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-06-11T13:29:56.158 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-06-11T13:29:56.158 DEBUG:teuthology.orchestra.run.smithi135:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-06-11T13:29:56.227 INFO:teuthology.orchestra.run.smithi135.stdout:ttyS1 2024-06-11T13:29:56.261 DEBUG:teuthology.parallel:result is None 2024-06-11T13:29:56.261 INFO:teuthology.run_tasks:Running task internal.base... 2024-06-11T13:29:56.270 INFO:teuthology.task.internal:Creating test directory... 2024-06-11T13:29:56.270 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-11T13:29:56.273 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-06-11T13:29:56.288 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-06-11T13:29:56.303 INFO:teuthology.run_tasks:Running task internal.archive... 2024-06-11T13:29:56.353 INFO:teuthology.task.internal:Creating archive directory... 2024-06-11T13:29:56.353 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-11T13:29:56.356 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-06-11T13:29:56.381 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-06-11T13:29:56.403 INFO:teuthology.task.internal:Enabling coredump saving... 2024-06-11T13:29:56.404 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-11T13:29:56.425 DEBUG:teuthology.orchestra.run.smithi135:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-06-11T13:29:56.472 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-11T13:29:56.482 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-11T13:29:56.484 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-11T13:29:56.494 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-06-11T13:29:56.495 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-06-11T13:29:56.505 INFO:teuthology.task.internal:Configuring sudo... 2024-06-11T13:29:56.505 DEBUG:teuthology.orchestra.run.smithi069:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-11T13:29:56.508 DEBUG:teuthology.orchestra.run.smithi135:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-06-11T13:29:56.534 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-06-11T13:29:56.544 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-06-11T13:29:56.544 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-11T13:29:56.577 DEBUG:teuthology.orchestra.run.smithi135:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-06-11T13:29:56.592 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-11T13:29:56.678 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-11T13:29:56.722 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-11T13:29:56.809 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-11T13:29:56.852 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:29:56.853 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-11T13:29:56.920 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-11T13:29:56.964 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-06-11T13:29:57.012 DEBUG:teuthology.orchestra.run.smithi135:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-11T13:29:57.094 DEBUG:teuthology.orchestra.run.smithi135:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-06-11T13:29:57.136 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-06-11T13:29:57.136 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-06-11T13:29:57.202 DEBUG:teuthology.orchestra.run.smithi069:> sudo service rsyslog restart 2024-06-11T13:29:57.204 DEBUG:teuthology.orchestra.run.smithi135:> sudo service rsyslog restart 2024-06-11T13:29:57.248 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-11T13:29:57.290 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-11T13:29:57.802 INFO:teuthology.run_tasks:Running task internal.timer... 2024-06-11T13:29:57.811 INFO:teuthology.task.internal:Starting timer... 2024-06-11T13:29:57.811 INFO:teuthology.run_tasks:Running task pcp... 2024-06-11T13:29:57.858 INFO:teuthology.run_tasks:Running task selinux... 2024-06-11T13:29:57.906 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-06-11T13:29:57.906 DEBUG:teuthology.orchestra.run.smithi069:> sudo service auditd rotate 2024-06-11T13:29:57.981 INFO:teuthology.orchestra.run.smithi069.stdout:Rotating logs: 2024-06-11T13:29:57.983 DEBUG:teuthology.orchestra.run.smithi135:> sudo service auditd rotate 2024-06-11T13:29:58.030 INFO:teuthology.orchestra.run.smithi135.stdout:Rotating logs: 2024-06-11T13:29:58.032 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-06-11T13:29:58.032 DEBUG:teuthology.orchestra.run.smithi069:> /usr/sbin/getenforce 2024-06-11T13:29:58.072 INFO:teuthology.orchestra.run.smithi069.stdout:Permissive 2024-06-11T13:29:58.072 DEBUG:teuthology.orchestra.run.smithi135:> /usr/sbin/getenforce 2024-06-11T13:29:58.091 INFO:teuthology.orchestra.run.smithi135.stdout:Permissive 2024-06-11T13:29:58.092 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi069.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi135.front.sepia.ceph.com': 'permissive'} 2024-06-11T13:29:58.092 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-11T13:29:58.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T13:29:58.144 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-11T13:29:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T13:29:58.169 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-06-11T13:29:58.169 DEBUG:teuthology.orchestra.run.smithi069:> sudo /usr/sbin/setenforce permissive 2024-06-11T13:29:58.214 DEBUG:teuthology.orchestra.run.smithi135:> sudo /usr/sbin/setenforce permissive 2024-06-11T13:29:58.239 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-06-11T13:29:58.247 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-06-11T13:29:58.250 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-06-11T13:29:58.366 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-06-11T13:29:58.377 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-06-11T13:29:58.378 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi069.front.sepia.ceph.com,smithi135.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-06-11T13:34:27.533 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi069.front.sepia.ceph.com'), Remote(name='ubuntu@smithi135.front.sepia.ceph.com')] 2024-06-11T13:34:27.535 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-06-11T13:34:27.536 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:34:27.615 DEBUG:teuthology.orchestra.run.smithi069:> true 2024-06-11T13:34:27.690 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-06-11T13:34:27.691 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-06-11T13:34:27.691 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi135.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-06-11T13:34:27.767 DEBUG:teuthology.orchestra.run.smithi135:> true 2024-06-11T13:34:27.851 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi135.front.sepia.ceph.com' 2024-06-11T13:34:27.851 INFO:teuthology.run_tasks:Running task clock... 2024-06-11T13:34:27.873 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-06-11T13:34:27.874 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-11T13:34:27.874 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-11T13:34:27.876 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-06-11T13:34:27.877 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-11T13:34:27.904 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-11T13:34:27.919 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-11T13:34:27.922 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-06-11T13:34:27.936 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-06-11T13:34:27.943 INFO:teuthology.orchestra.run.smithi069.stderr:sudo: ntpd: command not found 2024-06-11T13:34:27.954 INFO:teuthology.orchestra.run.smithi069.stdout:506 Cannot talk to daemon 2024-06-11T13:34:27.961 INFO:teuthology.orchestra.run.smithi135.stderr:sudo: ntpd: command not found 2024-06-11T13:34:27.967 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-11T13:34:27.972 INFO:teuthology.orchestra.run.smithi135.stdout:506 Cannot talk to daemon 2024-06-11T13:34:27.981 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-11T13:34:27.986 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-06-11T13:34:28.000 INFO:teuthology.orchestra.run.smithi135.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-06-11T13:34:28.036 INFO:teuthology.orchestra.run.smithi069.stderr:bash: line 1: ntpq: command not found 2024-06-11T13:34:28.040 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-11T13:34:28.040 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2024-06-11T13:34:28.040 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.041 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.041 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.041 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.056 INFO:teuthology.orchestra.run.smithi135.stderr:bash: line 1: ntpq: command not found 2024-06-11T13:34:28.060 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-11T13:34:28.060 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2024-06-11T13:34:28.060 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.060 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.060 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.060 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T13:34:28.061 INFO:teuthology.run_tasks:Running task cephadm... 2024-06-11T13:34:28.155 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '3f51ea3944d39ca13aa40153c9a4ee298306679d'} 2024-06-11T13:34:28.155 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-06-11T13:34:28.156 INFO:tasks.cephadm:Cluster fsid is 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:28.156 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-06-11T13:34:28.156 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-06-11T13:34:28.156 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi069': '172.21.15.69', 'mon.smithi135': '172.21.15.135'} 2024-06-11T13:34:28.156 INFO:tasks.cephadm:Normalizing hostnames... 2024-06-11T13:34:28.157 DEBUG:teuthology.orchestra.run.smithi069:> sudo hostname $(hostname -s) 2024-06-11T13:34:28.182 DEBUG:teuthology.orchestra.run.smithi135:> sudo hostname $(hostname -s) 2024-06-11T13:34:28.208 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-06-11T13:34:28.209 DEBUG:teuthology.orchestra.run.smithi069:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-11T13:34:28.465 INFO:teuthology.orchestra.run.smithi069.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 11 13:34 /home/ubuntu/cephtest/cephadm 2024-06-11T13:34:28.466 DEBUG:teuthology.orchestra.run.smithi135:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-06-11T13:34:28.634 INFO:teuthology.orchestra.run.smithi135.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Jun 11 13:34 /home/ubuntu/cephtest/cephadm 2024-06-11T13:34:28.634 DEBUG:teuthology.orchestra.run.smithi069:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-11T13:34:28.651 DEBUG:teuthology.orchestra.run.smithi135:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-06-11T13:34:28.674 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-06-11T13:34:28.675 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-11T13:34:28.693 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-06-11T13:34:29.527 INFO:teuthology.orchestra.run.smithi135.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-11T13:34:29.588 INFO:teuthology.orchestra.run.smithi069.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-11T13:34:46.608 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: "repo_digests": [ 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2024-06-11T13:34:46.609 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout:{ 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout: "repo_digests": [ 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout: ] 2024-06-11T13:34:47.185 INFO:teuthology.orchestra.run.smithi135.stdout:} 2024-06-11T13:34:47.203 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /etc/ceph 2024-06-11T13:34:47.231 DEBUG:teuthology.orchestra.run.smithi135:> sudo mkdir -p /etc/ceph 2024-06-11T13:34:47.258 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 777 /etc/ceph 2024-06-11T13:34:47.296 DEBUG:teuthology.orchestra.run.smithi135:> sudo chmod 777 /etc/ceph 2024-06-11T13:34:47.322 INFO:tasks.cephadm:Writing seed config... 2024-06-11T13:34:47.323 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-06-11T13:34:47.324 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-06-11T13:34:47.325 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:34:47.325 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-06-11T13:34:47.352 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 5368e33c-27f7-11ef-bc9b-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-06-11T13:34:47.352 DEBUG:teuthology.orchestra.run.smithi069:mon.smithi069> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service 2024-06-11T13:34:47.395 INFO:tasks.cephadm:Bootstrapping... 2024-06-11T13:34:47.395 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.69 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-06-11T13:34:47.589 INFO:teuthology.orchestra.run.smithi069.stderr:-------------------------------------------------------------------------------- 2024-06-11T13:34:47.589 INFO:teuthology.orchestra.run.smithi069.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '5368e33c-27f7-11ef-bc9b-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.69', '--skip-admin-label'] 2024-06-11T13:34:47.612 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 5.0.2 2024-06-11T13:34:47.614 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying podman|docker is present... 2024-06-11T13:34:47.636 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 5.0.2 2024-06-11T13:34:47.637 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying lvm2 is present... 2024-06-11T13:34:47.638 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying time synchronization is in place... 2024-06-11T13:34:47.645 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-11T13:34:47.652 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-06-11T13:34:47.658 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: enabled 2024-06-11T13:34:47.665 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: active 2024-06-11T13:34:47.665 INFO:teuthology.orchestra.run.smithi069.stderr:Unit chronyd.service is enabled and running 2024-06-11T13:34:47.665 INFO:teuthology.orchestra.run.smithi069.stderr:Repeating the final host check... 2024-06-11T13:34:47.686 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 5.0.2 2024-06-11T13:34:47.688 INFO:teuthology.orchestra.run.smithi069.stderr:podman (/bin/podman) version 5.0.2 is present 2024-06-11T13:34:47.688 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl is present 2024-06-11T13:34:47.689 INFO:teuthology.orchestra.run.smithi069.stderr:lvcreate is present 2024-06-11T13:34:47.694 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-06-11T13:34:47.701 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-06-11T13:34:47.708 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: enabled 2024-06-11T13:34:47.714 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: active 2024-06-11T13:34:47.715 INFO:teuthology.orchestra.run.smithi069.stderr:Unit chronyd.service is enabled and running 2024-06-11T13:34:47.715 INFO:teuthology.orchestra.run.smithi069.stderr:Host looks OK 2024-06-11T13:34:47.715 INFO:teuthology.orchestra.run.smithi069.stderr:Cluster fsid: 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:47.715 INFO:teuthology.orchestra.run.smithi069.stderr:Acquiring lock 140699883749232 on /run/cephadm/5368e33c-27f7-11ef-bc9b-c7b262605968.lock 2024-06-11T13:34:47.715 INFO:teuthology.orchestra.run.smithi069.stderr:Lock 140699883749232 acquired on /run/cephadm/5368e33c-27f7-11ef-bc9b-c7b262605968.lock 2024-06-11T13:34:47.715 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying IP 172.21.15.69 port 3300 ... 2024-06-11T13:34:47.716 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying IP 172.21.15.69 port 6789 ... 2024-06-11T13:34:47.716 INFO:teuthology.orchestra.run.smithi069.stderr:Base mon IP is 172.21.15.69, final addrv is [v2:172.21.15.69:3300,v1:172.21.15.69:6789] 2024-06-11T13:34:47.718 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.69 metric 100 2024-06-11T13:34:47.718 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.69 metric 100 2024-06-11T13:34:47.721 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-06-11T13:34:47.721 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-06-11T13:34:47.721 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1797sec hoplimit 64 pref medium 2024-06-11T13:34:47.724 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-06-11T13:34:47.724 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-06-11T13:34:47.724 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-11T13:34:47.724 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-06-11T13:34:47.724 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d737/64 scope link 2024-06-11T13:34:47.724 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-06-11T13:34:47.725 INFO:teuthology.orchestra.run.smithi069.stderr:Mon IP `172.21.15.69` is in CIDR network `172.21.0.0/20` 2024-06-11T13:34:47.725 INFO:teuthology.orchestra.run.smithi069.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-06-11T13:34:47.725 INFO:teuthology.orchestra.run.smithi069.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-06-11T13:34:47.826 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Getting image source signatures 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-06-11T13:34:48.270 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-11T13:34:48.430 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Writing manifest to image destination 2024-06-11T13:34:48.582 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-06-11T13:34:48.940 INFO:teuthology.orchestra.run.smithi069.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-11T13:34:49.481 INFO:teuthology.orchestra.run.smithi069.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-06-11T13:34:49.481 INFO:teuthology.orchestra.run.smithi069.stderr:Extracting ceph user uid/gid from container image... 2024-06-11T13:34:49.872 INFO:teuthology.orchestra.run.smithi069.stderr:stat: 167 167 2024-06-11T13:34:50.344 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial keys... 2024-06-11T13:34:50.702 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQB6UmhmPiV4JRAAkPoMX5BaxyoG/9F/l/m/iA== 2024-06-11T13:34:51.708 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQB7UmhmeiPbJhAAJyOp2xWSFRoewrutjHK6jA== 2024-06-11T13:34:52.697 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQB8UmhmfKRRJRAAyCZmy8FW8dVG2Jo/k7Lrmg== 2024-06-11T13:34:53.196 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial monmap... 2024-06-11T13:34:53.553 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-11T13:34:53.553 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-06-11T13:34:53.553 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:53.553 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-11T13:34:54.067 INFO:teuthology.orchestra.run.smithi069.stderr:monmaptool for smithi069 [v2:172.21.15.69:3300,v1:172.21.15.69:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-06-11T13:34:54.067 INFO:teuthology.orchestra.run.smithi069.stderr:setting min_mon_release = octopus 2024-06-11T13:34:54.067 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: set fsid to 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:54.067 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-06-11T13:34:54.067 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:34:54.068 INFO:teuthology.orchestra.run.smithi069.stderr:Creating mon... 2024-06-11T13:34:54.914 INFO:teuthology.orchestra.run.smithi069.stderr:create mon.smithi069 on 2024-06-11T13:34:55.108 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-06-11T13:34:55.252 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968.target → /etc/systemd/system/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968.target. 2024-06-11T13:34:55.253 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968.target → /etc/systemd/system/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968.target. 2024-06-11T13:34:55.553 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to reset failed state of unit ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service: Unit ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service not loaded. 2024-06-11T13:34:55.560 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968.target.wants/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service → /etc/systemd/system/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@.service. 2024-06-11T13:34:56.183 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:56 smithi069 podman[24696]: 2024-06-11 13:34:56.141913694 +0000 UTC m=+0.308380106 container start 13b131a458e0eb2fe3c378c3e5aef378251d6f7cc35d51cf16ea5477ed0c7f36 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, com.redhat.component=centos-stream-container, name=centos-stream, io.openshift.expose-services=, distribution-scope=public, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., architecture=x86_64, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-06-11T13:34:56.244 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-06-11T13:34:56.251 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-06-11T13:34:56.252 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-06-11T13:34:56.252 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to enable service . firewalld.service is not available 2024-06-11T13:34:56.252 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mon to start... 2024-06-11T13:34:56.252 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mon... 2024-06-11T13:34:56.450 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:56 smithi069 ceph-mon[24715]: mkfs 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:56.451 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:56 smithi069 ceph-mon[24715]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-06-11T13:34:56.451 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:56 smithi069 bash[24696]: 13b131a458e0eb2fe3c378c3e5aef378251d6f7cc35d51cf16ea5477ed0c7f36 2024-06-11T13:34:56.451 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:56 smithi069 systemd[1]: Started Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: cluster: 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: id: 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: services: 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi069 (age 0.542677s) 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr: no daemons active 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: data: 2024-06-11T13:34:56.724 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-06-11T13:34:56.725 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-06-11T13:34:56.725 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-06-11T13:34:56.725 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: pgs: 2024-06-11T13:34:56.725 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:34:57.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:57 smithi069 ceph-mon[24715]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-06-11T13:34:57.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:57 smithi069 ceph-mon[24715]: monmap e1: 1 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0]} 2024-06-11T13:34:57.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:57 smithi069 ceph-mon[24715]: fsmap 2024-06-11T13:34:57.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:57 smithi069 ceph-mon[24715]: osdmap e1: 0 total, 0 up, 0 in 2024-06-11T13:34:57.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:57 smithi069 ceph-mon[24715]: mgrmap e1: no daemons active 2024-06-11T13:34:57.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:57 smithi069 ceph-mon[24715]: from='client.? 172.21.15.69:0/174692776' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-06-11T13:34:57.273 INFO:teuthology.orchestra.run.smithi069.stderr:mon is available 2024-06-11T13:34:57.273 INFO:teuthology.orchestra.run.smithi069.stderr:Assimilating anything we can from ceph.conf... 2024-06-11T13:34:57.782 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:34:57.782 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [global] 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: fsid = 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.69:3300,v1:172.21.15.69:6789] 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [mgr] 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:34:57.783 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [osd] 2024-06-11T13:34:57.784 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-11T13:34:57.784 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-11T13:34:58.321 INFO:teuthology.orchestra.run.smithi069.stderr:Generating new minimal ceph.conf... 2024-06-11T13:34:58.442 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:58 smithi069 ceph-mon[24715]: from='client.? 172.21.15.69:0/2736589366' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-06-11T13:34:58.442 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:58 smithi069 ceph-mon[24715]: from='client.? 172.21.15.69:0/2736589366' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-06-11T13:34:59.289 INFO:teuthology.orchestra.run.smithi069.stderr:Restarting the monitor... 2024-06-11T13:34:59.307 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 ceph-mon[24715]: from='client.? 172.21.15.69:0/2262114660' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:34:59.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T13:34:59.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 bash[24934]: Error: no container with name or ID "ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon.smithi069" found: no such container 2024-06-11T13:34:59.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[24711]: 2024-06-11T13:34:59.450+0000 7faa845f2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T13:34:59.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[24711]: 2024-06-11T13:34:59.450+0000 7faa845f2700 -1 mon.smithi069@0(leader) e1 *** Got Signal Terminated *** 2024-06-11T13:34:59.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 podman[24945]: 2024-06-11 13:34:59.46128254 +0000 UTC m=+0.114897538 container died 13b131a458e0eb2fe3c378c3e5aef378251d6f7cc35d51cf16ea5477ed0c7f36 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, release=754) 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 podman[24945]: 2024-06-11 13:34:59.74988084 +0000 UTC m=+0.403495838 container cleanup 13b131a458e0eb2fe3c378c3e5aef378251d6f7cc35d51cf16ea5477ed0c7f36 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, architecture=x86_64, GIT_CLEAN=True, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, name=centos-stream, release=754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc.) 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 bash[24945]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 bash[24971]: Error: no container with name or ID "ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon.smithi069" found: no such container 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 podman[24957]: 2024-06-11 13:34:59.992660333 +0000 UTC m=+0.526860251 container remove 13b131a458e0eb2fe3c378c3e5aef378251d6f7cc35d51cf16ea5477ed0c7f36 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-type=git, version=8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, ceph=True, RELEASE=HEAD, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service: Deactivated successfully. 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:34:59 smithi069 systemd[1]: Stopped Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:35:00.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 systemd[1]: Starting Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T13:35:00.313 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 podman[25035]: 2024-06-11 13:35:00.218526586 +0000 UTC m=+0.105923073 container create 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, maintainer=Guillaume Abrioux , release=754) 2024-06-11T13:35:00.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 podman[25035]: 2024-06-11 13:35:00.130407871 +0000 UTC m=+0.017804377 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-06-11T13:35:00.479 INFO:teuthology.orchestra.run.smithi069.stderr:Setting mon public_network to 172.21.0.0/20 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 podman[25035]: 2024-06-11 13:35:00.412499393 +0000 UTC m=+0.299895882 container init 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, io.openshift.expose-services=, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vendor=Red Hat, Inc., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git) 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 podman[25035]: 2024-06-11 13:35:00.415658033 +0000 UTC m=+0.303054520 container start 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, ceph=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: set uid:gid to 167:167 (ceph:ceph) 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: pidfile_write: ignore empty --pid-file 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: load: jerasure load: lrc 2024-06-11T13:35:00.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: RocksDB version: 6.15.5 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Compile date Apr 18 2022 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: DB SUMMARY 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: DB Session ID: F73HL2218IEDO70OPXVW 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: CURRENT file: CURRENT 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: IDENTITY file: IDENTITY 2024-06-11T13:35:00.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi069/store.db dir, Total Num: 1, files: 000008.sst 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi069/store.db: 000010.log size: 82711 ; 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.error_if_exists: 0 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.create_if_missing: 0 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.paranoid_checks: 1 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.env: 0x55f2bcec6860 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.fs: Posix File System 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.info_log: 0x55f2be625fa0 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_file_opening_threads: 16 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.statistics: (nil) 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.use_fsync: 0 2024-06-11T13:35:00.696 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_log_file_size: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.keep_log_file_num: 1000 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.recycle_log_file_num: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.allow_fallocate: 1 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.allow_mmap_reads: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.allow_mmap_writes: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.use_direct_reads: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.create_missing_column_families: 0 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.db_log_dir: 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi069/store.db 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-11T13:35:00.697 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.advise_random_on_open: 1 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.db_write_buffer_size: 0 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.write_buffer_manager: 0x55f2be716330 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.rate_limiter: (nil) 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-11T13:35:00.698 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.wal_recovery_mode: 2 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.enable_thread_tracking: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.enable_pipelined_write: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.unordered_write: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.row_cache: None 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.wal_filter: None 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.allow_ingest_behind: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.preserve_deletes: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.two_write_queues: 0 2024-06-11T13:35:00.699 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.manual_wal_flush: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.atomic_flush: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.log_readahead_size: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.best_efforts_recovery: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.allow_data_in_errors: 0 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.db_host_id: __hostname__ 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_background_jobs: 2 2024-06-11T13:35:00.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_background_compactions: -1 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_subcompactions: 1 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_total_wal_size: 0 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_open_files: -1 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bytes_per_sync: 0 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-11T13:35:00.701 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_readahead_size: 0 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_background_flushes: -1 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Compression algorithms supported: 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kZSTD supported: 0 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kXpressCompression supported: 0 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kLZ4HCCompression supported: 1 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kLZ4Compression supported: 1 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kBZip2Compression supported: 0 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kZlibCompression supported: 1 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: kSnappyCompression supported: 1 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000009 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-11T13:35:00.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.merge_operator: 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_filter: None 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_filter_factory: None 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.sst_partitioner_factory: None 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f2be5f1d00) 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks: 1 2024-06-11T13:35:00.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_top_level_index_and_filter: 1 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_type: 0 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_index_type: 0 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_shortening: 1 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: hash_index_allow_collision: 1 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: checksum: 1 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: no_block_cache: 0 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache: 0x55f2be65c170 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_name: BinnedLRUCache 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_options: 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: capacity : 536870912 2024-06-11T13:35:00.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_shard_bits : 4 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: strict_capacity_limit : 0 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: high_pri_pool_ratio: 0.000 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_compressed: (nil) 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: persistent_cache: (nil) 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size: 4096 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size_deviation: 10 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_restart_interval: 16 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_block_restart_interval: 1 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_block_size: 4096 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: partition_filters: 0 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: use_delta_encoding: 1 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: whole_key_filtering: 1 2024-06-11T13:35:00.705 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: verify_compression: 0 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: read_amp_bytes_per_bit: 0 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: format_version: 4 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_index_compression: 1 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_align: 0 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.write_buffer_size: 33554432 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_write_buffer_number: 2 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression: NoCompression 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression: Disabled 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.prefix_extractor: nullptr 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.num_levels: 7 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-11T13:35:00.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.level: 32767 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.strategy: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:35:00.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-11T13:35:00.708 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compression_opts.enabled: false 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.target_file_size_base: 67108864 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-11T13:35:00.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.arena_block_size: 4194304 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.disable_auto_compactions: 0 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-11T13:35:00.710 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.table_properties_collectors: 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.inplace_update_support: 0 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.bloom_locality: 0 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.max_successive_merges: 0 2024-06-11T13:35:00.711 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.paranoid_file_checks: 0 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.force_consistency_checks: 1 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.report_bg_io_stats: 0 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.ttl: 2592000 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.enable_blob_files: false 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.min_blob_size: 0 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.blob_file_size: 268435456 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-06-11T13:35:00.712 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718112900441201, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718112900441883, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79393, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77657, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71966, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1718112900, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5f0d5c3a-0c28-4198-beae-7d8097c8189b", "db_session_id": "F73HL2218IEDO70OPXVW"}} 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718112900442215, "job": 1, "event": "recovery_finished"} 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi069/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f2be642a80 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: DB pointer 0x55f2be652000 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** DB Stats ** 2024-06-11T13:35:00.713 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-11T13:35:00.715 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 25.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 25.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:35:00.716 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-11T13:35:00.717 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 25.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: starting mon.smithi069 rank 0 at public addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] at bind addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi069 fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???) e1 preinit fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).mds e1 new map 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).mds e1 print_map 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: e1 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: legacy client fscid: -1 2024-06-11T13:35:00.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: No filesystems configured 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 bash[25035]: 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 systemd[1]: Started Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: monmap e1: 1 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0]} 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: fsmap 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: osdmap e1: 0 total, 0 up, 0 in 2024-06-11T13:35:00.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:00 smithi069 ceph-mon[25054]: mgrmap e1: no daemons active 2024-06-11T13:35:01.515 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote config to /etc/ceph/ceph.conf 2024-06-11T13:35:01.515 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-06-11T13:35:01.515 INFO:teuthology.orchestra.run.smithi069.stderr:Creating mgr... 2024-06-11T13:35:01.515 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying port 9283 ... 2024-06-11T13:35:01.681 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to reset failed state of unit ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mgr.smithi069.iruzyw.service: Unit ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mgr.smithi069.iruzyw.service not loaded. 2024-06-11T13:35:01.687 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968.target.wants/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mgr.smithi069.iruzyw.service → /etc/systemd/system/ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@.service. 2024-06-11T13:35:02.150 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:01 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2201240808' entity='client.admin' 2024-06-11T13:35:02.348 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-06-11T13:35:02.356 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-06-11T13:35:02.356 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-06-11T13:35:02.356 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to enable service . firewalld.service is not available 2024-06-11T13:35:02.365 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-06-11T13:35:02.372 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-06-11T13:35:02.373 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-06-11T13:35:02.373 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-06-11T13:35:02.373 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr to start... 2024-06-11T13:35:02.373 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr... 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "5368e33c-27f7-11ef-bc9b-c7b262605968", 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-06-11T13:35:02.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-06-11T13:35:02.906 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.907 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:02.908 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:02.909 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-06-11T13:34:56.183437+0000", 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-11T13:35:02.910 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:03.191 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:02 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3262055283' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-11T13:35:03.438 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (1/15)... 2024-06-11T13:35:06.153 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "5368e33c-27f7-11ef-bc9b-c7b262605968", 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:06.154 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-11T13:35:06.155 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.156 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.157 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-06-11T13:35:06.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:06.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-06-11T13:34:56.183437+0000", 2024-06-11T13:35:06.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:06.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:06.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-11T13:35:06.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:06.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:06 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2271395838' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-11T13:35:06.754 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (2/15)... 2024-06-11T13:35:08.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:35:08.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: mgrmap e2: smithi069.iruzyw(active, starting, since 0.00260256s) 2024-06-11T13:35:08.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:35:08.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:35:08.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:35:08.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:08.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:08 smithi069 ceph-mon[25054]: from='mgr.14100 172.21.15.69:0/2721474028' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:09.241 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:09.241 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:09.241 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "5368e33c-27f7-11ef-bc9b-c7b262605968", 2024-06-11T13:35:09.241 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:09.242 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-11T13:35:09.243 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-11T13:35:09.247 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-11T13:35:09.248 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:09.249 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-06-11T13:34:56.183437+0000", 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-11T13:35:09.250 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:09.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:09 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/4291352127' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-11T13:35:09.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:09 smithi069 ceph-mon[25054]: mgrmap e3: smithi069.iruzyw(active, since 1.00641s) 2024-06-11T13:35:09.760 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (3/15)... 2024-06-11T13:35:11.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:11 smithi069 ceph-mon[25054]: mgrmap e4: smithi069.iruzyw(active, since 2s) 2024-06-11T13:35:15.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:15 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1694288276' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-06-11T13:35:16.025 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:16.025 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "5368e33c-27f7-11ef-bc9b-c7b262605968", 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-06-11T13:35:16.026 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 14, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-06-11T13:35:16.027 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-06-11T13:35:16.031 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-06-11T13:35:16.032 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-06-11T13:34:56.183437+0000", 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-06-11T13:35:16.033 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:20.379 INFO:teuthology.orchestra.run.smithi069.stderr:mgr is available 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [global] 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: fsid = 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [mgr] 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-06-11T13:35:21.008 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-06-11T13:35:21.009 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [osd] 2024-06-11T13:35:21.009 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-06-11T13:35:21.009 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-06-11T13:35:21.566 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:21 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2028688198' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-06-11T13:35:21.567 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling cephadm module... 2024-06-11T13:35:22.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:22 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3029365847' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-06-11T13:35:23.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:23 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3029365847' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-06-11T13:35:23.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:23 smithi069 ceph-mon[25054]: mgrmap e5: smithi069.iruzyw(active, since 14s) 2024-06-11T13:35:23.719 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:23.719 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 5, 2024-06-11T13:35:23.719 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-06-11T13:35:23.719 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "active_name": "smithi069.iruzyw", 2024-06-11T13:35:23.719 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-11T13:35:23.719 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:24.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:24 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3007407387' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-11T13:35:24.660 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for the mgr to restart... 2024-06-11T13:35:24.660 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr epoch 5... 2024-06-11T13:35:28.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:28 smithi069 ceph-mon[25054]: Active manager daemon smithi069.iruzyw restarted 2024-06-11T13:35:28.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:28 smithi069 ceph-mon[25054]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:35:28.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:28 smithi069 ceph-mon[25054]: osdmap e2: 0 total, 0 up, 0 in 2024-06-11T13:35:29.542 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:29.542 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-06-11T13:35:29.542 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "initialized": true 2024-06-11T13:35:29.542 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:29.898 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: mgrmap e6: smithi069.iruzyw(active, starting, since 0.0525319s) 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:35:29.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:29 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:35:30.184 INFO:teuthology.orchestra.run.smithi069.stderr:mgr epoch 5 is available 2024-06-11T13:35:30.185 INFO:teuthology.orchestra.run.smithi069.stderr:Setting orchestrator backend to cephadm... 2024-06-11T13:35:30.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:30 smithi069 ceph-mon[25054]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-06-11T13:35:30.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:30 smithi069 ceph-mon[25054]: mgrmap e7: smithi069.iruzyw(active, since 1.0593s) 2024-06-11T13:35:30.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:30 smithi069 ceph-mon[25054]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-06-11T13:35:31.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: [11/Jun/2024:13:35:30] ENGINE Bus STARTING 2024-06-11T13:35:31.945 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: [11/Jun/2024:13:35:30] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: [11/Jun/2024:13:35:30] ENGINE Bus STARTED 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:31.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:31 smithi069 ceph-mon[25054]: mgrmap e8: smithi069.iruzyw(active, since 2s) 2024-06-11T13:35:31.946 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: value unchanged 2024-06-11T13:35:32.425 INFO:teuthology.orchestra.run.smithi069.stderr:Generating ssh key... 2024-06-11T13:35:32.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:32 smithi069 ceph-mon[25054]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:34.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:34 smithi069 ceph-mon[25054]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:34.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:34 smithi069 ceph-mon[25054]: Generating ssh key... 2024-06-11T13:35:34.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:34 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:34.373 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:34 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:34.373 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpaVaxZMFUJdAuo8duyktbFSATm9i8vSFBaR9qNkmOOUsCMbolTtIKFfKsVNEror/NQBGEGTpj/+CtB6KkDJ5MM3RUrZOTEw7zsWyQ8+tlm0Zhc4q4l+b7uzHJI5Kfdo7TOlIQk01i760oAumeo7cGrMYUK5OqGpHsZ89QhsaTjVaqh0qR7MntW9w/TDnNBVwZ9V37jQd+XKqYzJUwoG71Xws/xWrl/A3mXrVOoDPy2/c9okyZdP9EFhEdRbyYO6dS2QpTTJnsGinASZ10TrjyO52kC20WJiMQdl/1L5/9Hc5ZKRNFjL1P+GVuxV3BCk+e7TPb0wx4RqoP6L0+4d7U7e/6yw4JVZDHar+T8fbVIrRjasrKu+0MBo/EKX0y2C1QxKN0bAlsGXWaQ39ZHnL6SROn1alAxj4ULJ8TwjCupkhb4s2YIu7F18wQy9ieom/Ccq/+9WhZG97IqkSvSYqvwX2ZnldMDTKJBl0kpoYGchMbQr5oB5g7j4tsOKEEm0c= ceph-5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:35:34.893 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-06-11T13:35:34.893 INFO:teuthology.orchestra.run.smithi069.stderr:Adding key to root@localhost authorized_keys... 2024-06-11T13:35:34.893 INFO:teuthology.orchestra.run.smithi069.stderr:Adding host smithi069... 2024-06-11T13:35:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:35 smithi069 ceph-mon[25054]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:36.329 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Added host 'smithi069' with addr '172.21.15.69' 2024-06-11T13:35:36.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:36 smithi069 ceph-mon[25054]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi069", "addr": "172.21.15.69", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:36.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:36 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:36.656 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:36 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:36.925 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying mon service with default placement... 2024-06-11T13:35:37.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:37 smithi069 ceph-mon[25054]: Deploying cephadm binary to smithi069 2024-06-11T13:35:37.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:37 smithi069 ceph-mon[25054]: Added host smithi069 2024-06-11T13:35:37.540 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled mon update... 2024-06-11T13:35:38.106 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying mgr service with default placement... 2024-06-11T13:35:38.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:38 smithi069 ceph-mon[25054]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:38.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:38 smithi069 ceph-mon[25054]: Saving service mon spec with placement count:5 2024-06-11T13:35:38.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:38 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:38.798 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-06-11T13:35:39.845 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:39 smithi069 ceph-mon[25054]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:39.845 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:39 smithi069 ceph-mon[25054]: Saving service mgr spec with placement count:2 2024-06-11T13:35:39.845 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:39 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:40.418 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying crash service with default placement... 2024-06-11T13:35:40.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:40 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:41.038 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled crash update... 2024-06-11T13:35:41.599 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying prometheus service with default placement... 2024-06-11T13:35:42.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:41 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:42.191 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:41 smithi069 ceph-mon[25054]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:42.191 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:41 smithi069 ceph-mon[25054]: Saving service crash spec with placement * 2024-06-11T13:35:42.191 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:41 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:42.191 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-06-11T13:35:43.194 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying grafana service with default placement... 2024-06-11T13:35:43.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:43 smithi069 ceph-mon[25054]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:43.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:43 smithi069 ceph-mon[25054]: Saving service prometheus spec with placement count:1 2024-06-11T13:35:43.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:43 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:43.846 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-06-11T13:35:44.490 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:44 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:44.491 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:44 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:44.491 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:44 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:44.493 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying node-exporter service with default placement... 2024-06-11T13:35:45.126 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-06-11T13:35:45.383 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:45 smithi069 ceph-mon[25054]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:45.383 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:45 smithi069 ceph-mon[25054]: Saving service grafana spec with placement count:1 2024-06-11T13:35:45.383 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:45 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:46.086 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying alertmanager service with default placement... 2024-06-11T13:35:46.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:46 smithi069 ceph-mon[25054]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:46.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:46 smithi069 ceph-mon[25054]: Saving service node-exporter spec with placement * 2024-06-11T13:35:47.061 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-06-11T13:35:48.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:48 smithi069 ceph-mon[25054]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:35:48.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:48 smithi069 ceph-mon[25054]: Saving service alertmanager spec with placement count:1 2024-06-11T13:35:48.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:48 smithi069 ceph-mon[25054]: from='mgr.14120 172.21.15.69:0/337181930' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:49.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:49 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1361629037' entity='client.admin' 2024-06-11T13:35:50.012 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling the dashboard module... 2024-06-11T13:35:50.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:50 smithi069 ceph-mon[25054]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:35:50.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:50 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1750617818' entity='client.admin' 2024-06-11T13:35:51.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:51 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2084506992' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-06-11T13:35:52.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:52 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2084506992' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-06-11T13:35:52.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:52 smithi069 ceph-mon[25054]: mgrmap e9: smithi069.iruzyw(active, since 22s) 2024-06-11T13:35:52.874 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:52.875 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 9, 2024-06-11T13:35:52.875 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-06-11T13:35:52.875 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "active_name": "smithi069.iruzyw", 2024-06-11T13:35:52.875 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standby": 0 2024-06-11T13:35:52.875 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:53.489 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for the mgr to restart... 2024-06-11T13:35:53.489 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr epoch 9... 2024-06-11T13:35:53.724 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:53 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3313634816' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-06-11T13:35:58.604 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-06-11T13:35:58.604 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-06-11T13:35:58.604 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "initialized": true 2024-06-11T13:35:58.604 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: Active manager daemon smithi069.iruzyw restarted 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: osdmap e3: 0 total, 0 up, 0 in 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: mgrmap e10: smithi069.iruzyw(active, starting, since 0.0551965s) 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:35:58.930 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:35:58.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:58.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:35:58.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:35:58.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:58 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:35:59.207 INFO:teuthology.orchestra.run.smithi069.stderr:mgr epoch 9 is available 2024-06-11T13:35:59.210 INFO:teuthology.orchestra.run.smithi069.stderr:Generating a dashboard self-signed certificate... 2024-06-11T13:35:59.903 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:59 smithi069 ceph-mon[25054]: mgrmap e11: smithi069.iruzyw(active, since 1.05806s) 2024-06-11T13:35:59.903 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:59 smithi069 ceph-mon[25054]: [11/Jun/2024:13:35:59] ENGINE Bus STARTING 2024-06-11T13:35:59.904 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:59 smithi069 ceph-mon[25054]: [11/Jun/2024:13:35:59] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:35:59.904 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:59 smithi069 ceph-mon[25054]: [11/Jun/2024:13:35:59] ENGINE Bus STARTED 2024-06-11T13:35:59.904 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:35:59 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:35:59.905 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Self-signed certificate created 2024-06-11T13:36:00.498 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial admin user... 2024-06-11T13:36:00.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:00 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:00.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:00 smithi069 ceph-mon[25054]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:36:00.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:00 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:00.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:00 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:00.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:00 smithi069 ceph-mon[25054]: mgrmap e12: smithi069.iruzyw(active, since 2s) 2024-06-11T13:36:00.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:00 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:01.370 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$yKiFpedxuUrbKXeF0I3tNeFDbtC96fm3ug2meuTy3ONqXDUU6w.bK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1718112961, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-06-11T13:36:02.626 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:02 smithi069 ceph-mon[25054]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:36:02.626 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:02 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:02.629 INFO:teuthology.orchestra.run.smithi069.stderr:Fetching dashboard port number... 2024-06-11T13:36:03.255 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 8443 2024-06-11T13:36:03.617 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:03 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3554429674' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-06-11T13:36:03.946 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-06-11T13:36:03.954 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-06-11T13:36:03.955 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-06-11T13:36:03.955 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-06-11T13:36:03.956 INFO:teuthology.orchestra.run.smithi069.stderr:Ceph Dashboard is now available at: 2024-06-11T13:36:03.956 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:03.956 INFO:teuthology.orchestra.run.smithi069.stderr: URL: https://smithi069.front.sepia.ceph.com:8443/ 2024-06-11T13:36:03.957 INFO:teuthology.orchestra.run.smithi069.stderr: User: admin 2024-06-11T13:36:03.957 INFO:teuthology.orchestra.run.smithi069.stderr: Password: lesqgwpi7v 2024-06-11T13:36:03.957 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:03.957 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling autotune for osd_memory_target 2024-06-11T13:36:05.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:05 smithi069 ceph-mon[25054]: mgrmap e13: smithi069.iruzyw(active, since 6s) 2024-06-11T13:36:05.976 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr:You can access the Ceph CLI with: 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr:Please consider enabling telemetry to help improve Ceph: 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr: ceph telemetry on 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:06.643 INFO:teuthology.orchestra.run.smithi069.stderr:For more information see: 2024-06-11T13:36:06.644 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:06.644 INFO:teuthology.orchestra.run.smithi069.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-06-11T13:36:06.644 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T13:36:06.644 INFO:teuthology.orchestra.run.smithi069.stderr:Bootstrap complete. 2024-06-11T13:36:06.687 INFO:tasks.cephadm:Fetching config... 2024-06-11T13:36:06.688 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:36:06.688 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-06-11T13:36:06.706 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-06-11T13:36:06.706 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:36:06.706 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-06-11T13:36:06.762 INFO:tasks.cephadm:Fetching mon keyring... 2024-06-11T13:36:06.762 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:36:06.763 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/keyring of=/dev/stdout 2024-06-11T13:36:06.829 INFO:tasks.cephadm:Fetching pub ssh key... 2024-06-11T13:36:06.829 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:36:06.829 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-06-11T13:36:06.884 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-06-11T13:36:06.884 DEBUG:teuthology.orchestra.run.smithi069:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpaVaxZMFUJdAuo8duyktbFSATm9i8vSFBaR9qNkmOOUsCMbolTtIKFfKsVNEror/NQBGEGTpj/+CtB6KkDJ5MM3RUrZOTEw7zsWyQ8+tlm0Zhc4q4l+b7uzHJI5Kfdo7TOlIQk01i760oAumeo7cGrMYUK5OqGpHsZ89QhsaTjVaqh0qR7MntW9w/TDnNBVwZ9V37jQd+XKqYzJUwoG71Xws/xWrl/A3mXrVOoDPy2/c9okyZdP9EFhEdRbyYO6dS2QpTTJnsGinASZ10TrjyO52kC20WJiMQdl/1L5/9Hc5ZKRNFjL1P+GVuxV3BCk+e7TPb0wx4RqoP6L0+4d7U7e/6yw4JVZDHar+T8fbVIrRjasrKu+0MBo/EKX0y2C1QxKN0bAlsGXWaQ39ZHnL6SROn1alAxj4ULJ8TwjCupkhb4s2YIu7F18wQy9ieom/Ccq/+9WhZG97IqkSvSYqvwX2ZnldMDTKJBl0kpoYGchMbQr5oB5g7j4tsOKEEm0c= ceph-5368e33c-27f7-11ef-bc9b-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-11T13:36:06.960 INFO:teuthology.orchestra.run.smithi069.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpaVaxZMFUJdAuo8duyktbFSATm9i8vSFBaR9qNkmOOUsCMbolTtIKFfKsVNEror/NQBGEGTpj/+CtB6KkDJ5MM3RUrZOTEw7zsWyQ8+tlm0Zhc4q4l+b7uzHJI5Kfdo7TOlIQk01i760oAumeo7cGrMYUK5OqGpHsZ89QhsaTjVaqh0qR7MntW9w/TDnNBVwZ9V37jQd+XKqYzJUwoG71Xws/xWrl/A3mXrVOoDPy2/c9okyZdP9EFhEdRbyYO6dS2QpTTJnsGinASZ10TrjyO52kC20WJiMQdl/1L5/9Hc5ZKRNFjL1P+GVuxV3BCk+e7TPb0wx4RqoP6L0+4d7U7e/6yw4JVZDHar+T8fbVIrRjasrKu+0MBo/EKX0y2C1QxKN0bAlsGXWaQ39ZHnL6SROn1alAxj4ULJ8TwjCupkhb4s2YIu7F18wQy9ieom/Ccq/+9WhZG97IqkSvSYqvwX2ZnldMDTKJBl0kpoYGchMbQr5oB5g7j4tsOKEEm0c= ceph-5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:36:06.971 DEBUG:teuthology.orchestra.run.smithi135:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpaVaxZMFUJdAuo8duyktbFSATm9i8vSFBaR9qNkmOOUsCMbolTtIKFfKsVNEror/NQBGEGTpj/+CtB6KkDJ5MM3RUrZOTEw7zsWyQ8+tlm0Zhc4q4l+b7uzHJI5Kfdo7TOlIQk01i760oAumeo7cGrMYUK5OqGpHsZ89QhsaTjVaqh0qR7MntW9w/TDnNBVwZ9V37jQd+XKqYzJUwoG71Xws/xWrl/A3mXrVOoDPy2/c9okyZdP9EFhEdRbyYO6dS2QpTTJnsGinASZ10TrjyO52kC20WJiMQdl/1L5/9Hc5ZKRNFjL1P+GVuxV3BCk+e7TPb0wx4RqoP6L0+4d7U7e/6yw4JVZDHar+T8fbVIrRjasrKu+0MBo/EKX0y2C1QxKN0bAlsGXWaQ39ZHnL6SROn1alAxj4ULJ8TwjCupkhb4s2YIu7F18wQy9ieom/Ccq/+9WhZG97IqkSvSYqvwX2ZnldMDTKJBl0kpoYGchMbQr5oB5g7j4tsOKEEm0c= ceph-5368e33c-27f7-11ef-bc9b-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-06-11T13:36:07.013 INFO:teuthology.orchestra.run.smithi135.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpaVaxZMFUJdAuo8duyktbFSATm9i8vSFBaR9qNkmOOUsCMbolTtIKFfKsVNEror/NQBGEGTpj/+CtB6KkDJ5MM3RUrZOTEw7zsWyQ8+tlm0Zhc4q4l+b7uzHJI5Kfdo7TOlIQk01i760oAumeo7cGrMYUK5OqGpHsZ89QhsaTjVaqh0qR7MntW9w/TDnNBVwZ9V37jQd+XKqYzJUwoG71Xws/xWrl/A3mXrVOoDPy2/c9okyZdP9EFhEdRbyYO6dS2QpTTJnsGinASZ10TrjyO52kC20WJiMQdl/1L5/9Hc5ZKRNFjL1P+GVuxV3BCk+e7TPb0wx4RqoP6L0+4d7U7e/6yw4JVZDHar+T8fbVIrRjasrKu+0MBo/EKX0y2C1QxKN0bAlsGXWaQ39ZHnL6SROn1alAxj4ULJ8TwjCupkhb4s2YIu7F18wQy9ieom/Ccq/+9WhZG97IqkSvSYqvwX2ZnldMDTKJBl0kpoYGchMbQr5oB5g7j4tsOKEEm0c= ceph-5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:36:07.025 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-06-11T13:36:07.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:06 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2175263443' entity='client.admin' 2024-06-11T13:36:10.228 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-06-11T13:36:10.228 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-06-11T13:36:10.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:10 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1378688678' entity='client.admin' 2024-06-11T13:36:12.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:12 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:12.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:12 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:36:12.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:12 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:12.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:12 smithi069 ceph-mon[25054]: Deploying daemon alertmanager.smithi069 on smithi069 2024-06-11T13:36:13.445 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi135 2024-06-11T13:36:13.445 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-06-11T13:36:13.445 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/etc/ceph/ceph.conf 2024-06-11T13:36:13.464 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-06-11T13:36:13.464 DEBUG:teuthology.orchestra.run.smithi135:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:36:13.521 INFO:tasks.cephadm:Adding host smithi135 to orchestrator... 2024-06-11T13:36:13.522 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch host add smithi135 2024-06-11T13:36:13.981 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:13 smithi069 ceph-mon[25054]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:36:13.981 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:13 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:17.846 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:17 smithi069 ceph-mon[25054]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi135", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:36:17.846 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:17 smithi069 ceph-mon[25054]: Deploying cephadm binary to smithi135 2024-06-11T13:36:17.848 INFO:teuthology.orchestra.run.smithi069.stdout:Added host 'smithi135' with addr '172.21.15.135' 2024-06-11T13:36:18.991 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:18 smithi069 ceph-mon[25054]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:18.991 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:18 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:18.991 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:18 smithi069 ceph-mon[25054]: Added host smithi135 2024-06-11T13:36:19.425 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch host ls --format=json 2024-06-11T13:36:21.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:21 smithi069 ceph-mon[25054]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:21.777 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:36:21.777 INFO:teuthology.orchestra.run.smithi069.stdout:[{"addr": "172.21.15.69", "hostname": "smithi069", "labels": [], "status": ""}, {"addr": "172.21.15.135", "hostname": "smithi135", "labels": [], "status": ""}] 2024-06-11T13:36:22.923 INFO:tasks.cephadm:Setting crush tunables to default 2024-06-11T13:36:22.923 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd crush tunables default 2024-06-11T13:36:23.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:23.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:36:23.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:23.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:36:23.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-11T13:36:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:36:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: Deploying daemon crash.smithi069 on smithi069 2024-06-11T13:36:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:23 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:25.569 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:25 smithi069 ceph-mon[25054]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:26.481 INFO:teuthology.orchestra.run.smithi069.stderr:adjusted tunables profile to default 2024-06-11T13:36:26.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:26 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/254431554' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-06-11T13:36:26.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:26 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:26.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:26 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:26.719 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:26 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:26.720 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:26 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:36:26.720 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:26 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:26.960 INFO:tasks.cephadm:Adding mon.smithi069 on smithi069 2024-06-11T13:36:26.960 INFO:tasks.cephadm:Adding mon.smithi135 on smithi135 2024-06-11T13:36:26.960 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch apply mon '2;smithi069:172.21.15.69=smithi069;smithi135:172.21.15.135=smithi135' 2024-06-11T13:36:27.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:27 smithi069 ceph-mon[25054]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:27.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:27 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:36:27.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:27 smithi069 ceph-mon[25054]: Deploying daemon grafana.smithi069 on smithi069 2024-06-11T13:36:27.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:27 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/254431554' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-06-11T13:36:27.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:27 smithi069 ceph-mon[25054]: osdmap e4: 0 total, 0 up, 0 in 2024-06-11T13:36:28.693 INFO:teuthology.orchestra.run.smithi135.stdout:Scheduled mon update... 2024-06-11T13:36:28.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:28 smithi069 ceph-mon[25054]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:28.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:28 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:29.204 DEBUG:teuthology.orchestra.run.smithi135:mon.smithi135> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi135.service 2024-06-11T13:36:29.206 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:29.206 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:29.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:29 smithi069 ceph-mon[25054]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi069:172.21.15.69=smithi069;smithi135:172.21.15.135=smithi135", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:36:29.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:29 smithi069 ceph-mon[25054]: Saving service mon spec with placement smithi069:172.21.15.69=smithi069;smithi135:172.21.15.135=smithi135;count:2 2024-06-11T13:36:29.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:29 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:31.094 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:31.094 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:31.096 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:31.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:31 smithi069 ceph-mon[25054]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:31.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:31 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/4021393775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:32.662 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:32.662 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:33.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:33 smithi069 ceph-mon[25054]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:34.536 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:34.593 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:34.594 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:35.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:35 smithi069 ceph-mon[25054]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:35.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:35 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/4253839873' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:36.091 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:36.092 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:37.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:37 smithi069 ceph-mon[25054]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:37.832 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:37.832 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:37.834 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:38 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/1870480515' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:39.353 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:39.354 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:39.733 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:39 smithi069 ceph-mon[25054]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:40 smithi069 ceph-mon[25054]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:40 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:40 smithi069 ceph-mon[25054]: Deploying daemon node-exporter.smithi069 on smithi069 2024-06-11T13:36:41.177 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:41.177 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:41.179 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:42.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:41 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/4215780047' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:42.709 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:42.710 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:43.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:42 smithi069 ceph-mon[25054]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:36:43.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:42 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:44.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:43 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:44.207 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:43 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-11T13:36:44.461 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:44.461 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:44.463 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:45.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:44 smithi069 ceph-mon[25054]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-11T13:36:45.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:44 smithi069 ceph-mon[25054]: mgrmap e14: smithi069.iruzyw(active, since 46s) 2024-06-11T13:36:45.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:44 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/1360458022' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:45.988 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:45.989 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:47.864 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:47.865 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:47.866 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:47.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:47 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/1147043621' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:49.347 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:49.348 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:50.153 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:49 smithi069 ceph-mon[25054]: Active manager daemon smithi069.iruzyw restarted 2024-06-11T13:36:50.153 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:49 smithi069 ceph-mon[25054]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:36:50.153 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:49 smithi069 ceph-mon[25054]: osdmap e5: 0 total, 0 up, 0 in 2024-06-11T13:36:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: mgrmap e15: smithi069.iruzyw(active, starting, since 0.0529704s) 2024-06-11T13:36:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:36:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:36:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:50.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:51.337 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:51.337 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:51.339 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:52.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:51 smithi069 ceph-mon[25054]: [11/Jun/2024:13:36:50] ENGINE Bus STARTING 2024-06-11T13:36:52.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:51 smithi069 ceph-mon[25054]: [11/Jun/2024:13:36:50] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:36:52.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:51 smithi069 ceph-mon[25054]: [11/Jun/2024:13:36:50] ENGINE Bus STARTED 2024-06-11T13:36:52.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:51 smithi069 ceph-mon[25054]: mgrmap e16: smithi069.iruzyw(active, since 1.06052s) 2024-06-11T13:36:52.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:52.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:51 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2369294682' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:53.000 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:53.000 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:53.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:53.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:53 smithi069 ceph-mon[25054]: mgrmap e17: smithi069.iruzyw(active, since 2s) 2024-06-11T13:36:55.666 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:55.666 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:55.668 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:56 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2998791283' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:36:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:56 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:56 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:56 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd/host:smithi135", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:36:57.212 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:36:57.213 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:36:58.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:58.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:58.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:36:58.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:58.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:36:58.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:36:58.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-11T13:36:58.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:36:59.152 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:36:59.153 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:36:59.155 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:36:59.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:59 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:36:59.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:59 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:36:59.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:59 smithi069 ceph-mon[25054]: Deploying daemon crash.smithi135 on smithi135 2024-06-11T13:36:59.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:36:59 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2291559737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:01.033 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:37:01.033 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:37:02.801 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:37:02.801 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:37:02.803 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:37:03.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:02 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/3000589608' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:37:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-11T13:37:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:37:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:04.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:03 smithi069 ceph-mon[25054]: Deploying daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:37:04.518 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:37:04.518 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:37:06.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:06.314 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:37:06.314 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:37:06.316 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:37:07.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:06 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2835606015' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:07.900 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:37:07.900 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:37:08.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:08.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:37:08.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:08.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:08 smithi069 ceph-mon[25054]: Deploying daemon mon.smithi135 on smithi135 2024-06-11T13:37:10.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:10 smithi069 ceph-mon[25054]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:10.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:10 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:11.339 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 podman[27477]: 2024-06-11 13:37:11.224337271 +0000 UTC m=+0.025357990 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-06-11T13:37:11.340 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 podman[27477]: 2024-06-11 13:37:11.341187565 +0000 UTC m=+0.142208269 container create 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, name=centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.openshift.expose-services=) 2024-06-11T13:37:11.461 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:37:11.461 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":1,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:34:53.486112Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-06-11T13:37:11.462 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 1 2024-06-11T13:37:11.588 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 podman[27477]: 2024-06-11 13:37:11.560497235 +0000 UTC m=+0.361517955 container init 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-06-11T13:37:11.588 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 podman[27477]: 2024-06-11 13:37:11.564152994 +0000 UTC m=+0.365173704 container start 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, com.redhat.component=centos-stream-container, distribution-scope=public, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, vendor=Red Hat, Inc.) 2024-06-11T13:37:11.588 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: set uid:gid to 167:167 (ceph:ceph) 2024-06-11T13:37:11.589 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-06-11T13:37:11.589 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pidfile_write: ignore empty --pid-file 2024-06-11T13:37:11.589 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: load: jerasure load: lrc 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: RocksDB version: 6.15.5 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Compile date Apr 18 2022 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: DB SUMMARY 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: DB Session ID: 445GWT74GTQI9ZEQ8Q76 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: CURRENT file: CURRENT 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: IDENTITY file: IDENTITY 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi135/store.db dir, Total Num: 0, files: 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi135/store.db: 000004.log size: 511 ; 2024-06-11T13:37:11.590 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.error_if_exists: 0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.create_if_missing: 0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.paranoid_checks: 1 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.env: 0x55cbf08c6860 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.fs: Posix File System 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.info_log: 0x55cbf17cdfa0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_file_opening_threads: 16 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.statistics: (nil) 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.use_fsync: 0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_log_file_size: 0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.keep_log_file_num: 1000 2024-06-11T13:37:11.591 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.recycle_log_file_num: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.allow_fallocate: 1 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.allow_mmap_reads: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.allow_mmap_writes: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.use_direct_reads: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.create_missing_column_families: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.db_log_dir: 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi135/store.db 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-11T13:37:11.592 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.advise_random_on_open: 1 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.db_write_buffer_size: 0 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.write_buffer_manager: 0x55cbf18be330 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.rate_limiter: (nil) 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.wal_recovery_mode: 2 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.enable_thread_tracking: 0 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.enable_pipelined_write: 0 2024-06-11T13:37:11.593 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.unordered_write: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.row_cache: None 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.wal_filter: None 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.allow_ingest_behind: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.preserve_deletes: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.two_write_queues: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.manual_wal_flush: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.atomic_flush: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-11T13:37:11.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.log_readahead_size: 0 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.best_efforts_recovery: 0 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.allow_data_in_errors: 0 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.db_host_id: __hostname__ 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_background_jobs: 2 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_background_compactions: -1 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_subcompactions: 1 2024-06-11T13:37:11.595 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_total_wal_size: 0 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-11T13:37:11.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_open_files: -1 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bytes_per_sync: 0 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_readahead_size: 0 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_background_flushes: -1 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Compression algorithms supported: 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-11T13:37:11.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kZSTD supported: 0 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kXpressCompression supported: 0 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kLZ4HCCompression supported: 1 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kLZ4Compression supported: 1 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kBZip2Compression supported: 0 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kZlibCompression supported: 1 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: kSnappyCompression supported: 1 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi135/store.db/MANIFEST-000003 2024-06-11T13:37:11.598 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.merge_operator: 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_filter: None 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_filter_factory: None 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.sst_partitioner_factory: None 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cbf1799d00) 2024-06-11T13:37:11.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: cache_index_and_filter_blocks: 1 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: pin_top_level_index_and_filter: 1 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: index_type: 0 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: data_block_index_type: 0 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: index_shortening: 1 2024-06-11T13:37:11.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: hash_index_allow_collision: 1 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: checksum: 1 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: no_block_cache: 0 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache: 0x55cbf1804170 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache_name: BinnedLRUCache 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache_options: 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: capacity : 536870912 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: num_shard_bits : 4 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: strict_capacity_limit : 0 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: high_pri_pool_ratio: 0.000 2024-06-11T13:37:11.601 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache_compressed: (nil) 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: persistent_cache: (nil) 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_size: 4096 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_size_deviation: 10 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_restart_interval: 16 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: index_block_restart_interval: 1 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: metadata_block_size: 4096 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: partition_filters: 0 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: use_delta_encoding: 1 2024-06-11T13:37:11.602 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: whole_key_filtering: 1 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: verify_compression: 0 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: read_amp_bytes_per_bit: 0 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: format_version: 4 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: enable_index_compression: 1 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_align: 0 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.write_buffer_size: 33554432 2024-06-11T13:37:11.603 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_write_buffer_number: 2 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression: NoCompression 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression: Disabled 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.prefix_extractor: nullptr 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.num_levels: 7 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-11T13:37:11.604 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.level: 32767 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.strategy: 0 2024-06-11T13:37:11.605 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compression_opts.enabled: false 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-11T13:37:11.606 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.target_file_size_base: 67108864 2024-06-11T13:37:11.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:11 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/3167538447' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-11T13:37:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-11T13:37:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-11T13:37:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.arena_block_size: 4194304 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-11T13:37:12.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.disable_auto_compactions: 0 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-11T13:37:12.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.table_properties_collectors: 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.inplace_update_support: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.bloom_locality: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.max_successive_merges: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.paranoid_file_checks: 0 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.force_consistency_checks: 1 2024-06-11T13:37:12.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.report_bg_io_stats: 0 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.ttl: 2592000 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.enable_blob_files: false 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.min_blob_size: 0 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.blob_file_size: 268435456 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi135/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-06-11T13:37:12.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718113031590674, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718113031591101, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1718113031, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "457848ae-49a0-4194-a73c-ca7939b539dd", "db_session_id": "445GWT74GTQI9ZEQ8Q76"}} 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718113031591512, "job": 1, "event": "recovery_finished"} 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi135/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55cbf17eaa80 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: DB pointer 0x55cbf17fa000 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** DB Stats ** 2024-06-11T13:37:12.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** Compaction Stats [default] ** 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** Compaction Stats [default] ** 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-11T13:37:12.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:37:12.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:37:12.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** Compaction Stats [default] ** 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** Compaction Stats [default] ** 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-11T13:37:12.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135 does not exist in monmap, will attempt to join an existing cluster 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: using public_addr v2:172.21.15.135:0/0 -> [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: starting mon.smithi135 rank -1 at public addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] at bind addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi135 fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(???) e0 preinit fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:37:12.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).mds e1 new map 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).mds e1 print_map 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: e1 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: legacy client fscid: -1 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: No filesystems configured 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-06-11T13:37:12.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Added host smithi135 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Deploying daemon crash.smithi069 on smithi069 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/254431554' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-06-11T13:37:12.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Deploying daemon grafana.smithi069 on smithi069 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/254431554' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: osdmap e4: 0 total, 0 up, 0 in 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi069:172.21.15.69=smithi069;smithi135:172.21.15.135=smithi135", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Saving service mon spec with placement smithi069:172.21.15.69=smithi069;smithi135:172.21.15.135=smithi135;count:2 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/4021393775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/4253839873' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/1870480515' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Deploying daemon node-exporter.smithi069 on smithi069 2024-06-11T13:37:12.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/4215780047' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14162 172.21.15.69:0/3821976626' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mgrmap e14: smithi069.iruzyw(active, since 46s) 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/1360458022' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/1147043621' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Active manager daemon smithi069.iruzyw restarted 2024-06-11T13:37:12.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: osdmap e5: 0 total, 0 up, 0 in 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mgrmap e15: smithi069.iruzyw(active, starting, since 0.0529704s) 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:37:12.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: [11/Jun/2024:13:36:50] ENGINE Bus STARTING 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: [11/Jun/2024:13:36:50] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: [11/Jun/2024:13:36:50] ENGINE Bus STARTED 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mgrmap e16: smithi069.iruzyw(active, since 1.06052s) 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2369294682' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mgrmap e17: smithi069.iruzyw(active, since 2s) 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2998791283' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd/host:smithi135", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Deploying daemon crash.smithi135 on smithi135 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2291559737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/3000589608' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:37:12.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Deploying daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2835606015' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: Deploying daemon mon.smithi135 on smithi135 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/3167538447' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 ceph-mon[27511]: mon.smithi135@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 bash[27477]: 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf 2024-06-11T13:37:12.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:11 smithi135 systemd[1]: Started Ceph mon.smithi135 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:37:13.371 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-06-11T13:37:13.371 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mon dump -f json 2024-06-11T13:37:16.924 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: mon.smithi069 calling monitor election 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: mon.smithi135 calling monitor election 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:16.925 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: mon.smithi069 is new leader, mons smithi069,smithi135 in quorum (ranks 0,1) 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: monmap e2: 2 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0],smithi135=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: fsmap 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: osdmap e5: 0 total, 0 up, 0 in 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: mgrmap e17: smithi069.iruzyw(active, since 26s) 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould started 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: overall HEALTH_OK 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:16.926 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:16 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:37:16.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: mon.smithi069 calling monitor election 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: mon.smithi135 calling monitor election 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: mon.smithi069 is new leader, mons smithi069,smithi135 in quorum (ranks 0,1) 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: monmap e2: 2 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0],smithi135=[v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0]} 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: fsmap 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: osdmap e5: 0 total, 0 up, 0 in 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: mgrmap e17: smithi069.iruzyw(active, since 26s) 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould started 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: overall HEALTH_OK 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:16.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:16 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/3252993254' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:37:17.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:17 smithi069 ceph-mon[25054]: Deploying daemon node-exporter.smithi135 on smithi135 2024-06-11T13:37:17.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:17 smithi069 ceph-mon[25054]: mgrmap e18: smithi069.iruzyw(active, since 26s), standbys: smithi135.xcould 2024-06-11T13:37:17.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:17 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:37:17.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:17 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:18.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:17 smithi135 ceph-mon[27511]: Deploying daemon node-exporter.smithi135 on smithi135 2024-06-11T13:37:18.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:17 smithi135 ceph-mon[27511]: mgrmap e18: smithi069.iruzyw(active, since 26s), standbys: smithi135.xcould 2024-06-11T13:37:18.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:17 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:37:18.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:17 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:37:18.221 INFO:teuthology.orchestra.run.smithi135.stdout: 2024-06-11T13:37:18.221 INFO:teuthology.orchestra.run.smithi135.stdout:{"epoch":2,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","modified":"2024-06-11T13:37:11.616682Z","created":"2024-06-11T13:34:53.486112Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi135","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:3300","nonce":0},{"type":"v1","addr":"172.21.15.135:6789","nonce":0}]},"addr":"172.21.15.135:6789/0","public_addr":"172.21.15.135:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-06-11T13:37:18.222 INFO:teuthology.orchestra.run.smithi135.stderr:dumped monmap epoch 2 2024-06-11T13:37:18.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:18 smithi069 ceph-mon[25054]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:18.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:18 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/560763726' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:19.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:18 smithi135 ceph-mon[27511]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:19.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:18 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/560763726' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-06-11T13:37:19.451 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-06-11T13:37:19.451 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph config generate-minimal-conf 2024-06-11T13:37:20.193 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:20 smithi135 ceph-mon[27511]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:20.194 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:20 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:20.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:20 smithi069 ceph-mon[25054]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:20.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:20 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:21.284 INFO:teuthology.orchestra.run.smithi069.stdout:# minimal ceph.conf for 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:37:21.284 INFO:teuthology.orchestra.run.smithi069.stdout:[global] 2024-06-11T13:37:21.285 INFO:teuthology.orchestra.run.smithi069.stdout: fsid = 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:37:21.285 INFO:teuthology.orchestra.run.smithi069.stdout: mon_host = [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] 2024-06-11T13:37:21.545 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:21.545 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:21 smithi135 ceph-mon[27511]: Deploying daemon prometheus.smithi069 on smithi069 2024-06-11T13:37:21.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:21.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:21 smithi069 ceph-mon[25054]: Deploying daemon prometheus.smithi069 on smithi069 2024-06-11T13:37:21.994 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-06-11T13:37:21.995 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:37:21.995 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.conf 2024-06-11T13:37:22.022 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:37:22.022 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:37:22.086 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-06-11T13:37:22.087 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.conf 2024-06-11T13:37:22.120 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-06-11T13:37:22.120 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:37:22.190 INFO:tasks.cephadm:Deploying OSDs... 2024-06-11T13:37:22.190 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:37:22.190 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/scratch_devs of=/dev/stdout 2024-06-11T13:37:22.205 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-11T13:37:22.206 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_1 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-06-11 13:36:40.408343479 +0000 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-06-11 13:33:36.583311505 +0000 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-06-11 13:33:36.583311505 +0000 2024-06-11T13:37:22.262 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-06-11 13:33:36.583311505 +0000 2024-06-11T13:37:22.263 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-11T13:37:22.325 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-06-11T13:37:22.325 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-06-11T13:37:22.325 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000194026 s, 2.6 MB/s 2024-06-11T13:37:22.326 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-11T13:37:22.383 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_2 2024-06-11T13:37:22.438 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-06-11 13:36:40.408343479 +0000 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-06-11 13:33:36.922303372 +0000 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-06-11 13:33:36.922303372 +0000 2024-06-11T13:37:22.439 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-06-11 13:33:36.922303372 +0000 2024-06-11T13:37:22.439 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-11T13:37:22.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:22 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1374570214' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:22.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:22 smithi069 ceph-mon[25054]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:22.504 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-06-11T13:37:22.504 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-06-11T13:37:22.504 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000174107 s, 2.9 MB/s 2024-06-11T13:37:22.506 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-11T13:37:22.545 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:22 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1374570214' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:22.545 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:22 smithi135 ceph-mon[27511]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:22.563 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_3 2024-06-11T13:37:22.619 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-11T13:37:22.619 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:22.619 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-06-11T13:37:22.619 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:22.619 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:22.620 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-06-11 13:36:40.408343479 +0000 2024-06-11T13:37:22.620 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-06-11 13:33:37.242295695 +0000 2024-06-11T13:37:22.620 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-06-11 13:33:37.242295695 +0000 2024-06-11T13:37:22.620 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-06-11 13:33:37.242295695 +0000 2024-06-11T13:37:22.620 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-11T13:37:22.683 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-06-11T13:37:22.683 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-06-11T13:37:22.684 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000163071 s, 3.1 MB/s 2024-06-11T13:37:22.685 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-11T13:37:22.740 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_4 2024-06-11T13:37:22.796 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-06-11 13:36:40.409343456 +0000 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-06-11 13:33:37.582287538 +0000 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-06-11 13:33:37.582287538 +0000 2024-06-11T13:37:22.797 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-06-11 13:33:37.582287538 +0000 2024-06-11T13:37:22.797 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-11T13:37:22.863 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-06-11T13:37:22.863 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-06-11T13:37:22.863 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000172679 s, 3.0 MB/s 2024-06-11T13:37:22.864 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-11T13:37:22.923 DEBUG:teuthology.orchestra.run.smithi135:> set -ex 2024-06-11T13:37:22.923 DEBUG:teuthology.orchestra.run.smithi135:> dd if=/scratch_devs of=/dev/stdout 2024-06-11T13:37:22.943 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-06-11T13:37:22.943 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_1 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-06-11 13:37:17.213081577 +0000 2024-06-11T13:37:23.002 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-06-11 13:33:47.127586134 +0000 2024-06-11T13:37:23.003 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-06-11 13:33:47.127586134 +0000 2024-06-11T13:37:23.003 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-06-11 13:33:47.127586134 +0000 2024-06-11T13:37:23.003 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-06-11T13:37:23.069 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-06-11T13:37:23.069 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-06-11T13:37:23.069 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000288718 s, 1.8 MB/s 2024-06-11T13:37:23.070 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-06-11T13:37:23.128 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_2 2024-06-11T13:37:23.185 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-06-11 13:37:17.213081577 +0000 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-06-11 13:33:47.479578306 +0000 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-06-11 13:33:47.479578306 +0000 2024-06-11T13:37:23.186 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-06-11 13:33:47.479578306 +0000 2024-06-11T13:37:23.187 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-06-11T13:37:23.250 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-06-11T13:37:23.251 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-06-11T13:37:23.251 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000186335 s, 2.7 MB/s 2024-06-11T13:37:23.252 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-06-11T13:37:23.309 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_3 2024-06-11T13:37:23.364 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-06-11 13:37:17.214081555 +0000 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-06-11 13:33:47.825570610 +0000 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-06-11 13:33:47.825570610 +0000 2024-06-11T13:37:23.365 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-06-11 13:33:47.825570610 +0000 2024-06-11T13:37:23.365 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-06-11T13:37:23.429 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-06-11T13:37:23.429 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-06-11T13:37:23.429 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000204834 s, 2.5 MB/s 2024-06-11T13:37:23.430 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-06-11T13:37:23.487 DEBUG:teuthology.orchestra.run.smithi135:> stat /dev/vg_nvme/lv_4 2024-06-11T13:37:23.542 INFO:teuthology.orchestra.run.smithi135.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout:Context: system_u:object_r:device_t:s0 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout:Access: 2024-06-11 13:37:17.214081555 +0000 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout:Modify: 2024-06-11 13:33:48.149563405 +0000 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout:Change: 2024-06-11 13:33:48.149563405 +0000 2024-06-11T13:37:23.543 INFO:teuthology.orchestra.run.smithi135.stdout: Birth: 2024-06-11 13:33:48.149563405 +0000 2024-06-11T13:37:23.543 DEBUG:teuthology.orchestra.run.smithi135:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-06-11T13:37:23.606 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records in 2024-06-11T13:37:23.606 INFO:teuthology.orchestra.run.smithi135.stderr:1+0 records out 2024-06-11T13:37:23.606 INFO:teuthology.orchestra.run.smithi135.stderr:512 bytes copied, 0.000194131 s, 2.6 MB/s 2024-06-11T13:37:23.607 DEBUG:teuthology.orchestra.run.smithi135:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-06-11T13:37:23.664 INFO:tasks.cephadm:Deploying osd.0 on smithi069 with /dev/vg_nvme/lv_4... 2024-06-11T13:37:23.664 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-11T13:37:24.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:24 smithi069 ceph-mon[25054]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:24.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:24 smithi135 ceph-mon[27511]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:26.141 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-11T13:37:26.141 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-11T13:37:26.141 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-06-11T13:37:26.141 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-06-11T13:37:26.141 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133064 s, 788 MB/s 2024-06-11T13:37:26.141 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-06-11T13:37:26.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:26 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:26.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:26 smithi135 ceph-mon[27511]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:26.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:26 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:26.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:26 smithi069 ceph-mon[25054]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:28.142 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_4 2024-06-11T13:37:28.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:28 smithi135 ceph-mon[27511]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:28.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:28 smithi069 ceph-mon[25054]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:30.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:30 smithi135 ceph-mon[27511]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:30.457 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:30 smithi069 ceph-mon[25054]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:32.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:32 smithi069 ceph-mon[25054]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:32.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:32 smithi135 ceph-mon[27511]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:33.846 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:37:33.846 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:37:33.846 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:33.884 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:37:33.884 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:37:33.885 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:35.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:34 smithi069 ceph-mon[25054]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:37:35.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:34 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:35.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:34 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:37:35.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:34 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:35.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:34 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:37:35.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:34 smithi069 ceph-mon[25054]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:35.224 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:34 smithi135 ceph-mon[27511]: from='client.14240 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:37:35.224 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:34 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:35.224 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:34 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:37:35.224 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:34 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:35.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:34 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:37:35.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:34 smithi135 ceph-mon[27511]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:36.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3777781316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4314153a-0a5d-44cb-b4fb-9e7e1c02118d"}]: dispatch 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3777781316' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4314153a-0a5d-44cb-b4fb-9e7e1c02118d"}]': finished 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: osdmap e6: 1 total, 0 up, 1 in 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:36 smithi135 ceph-mon[27511]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:36.323 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:36.323 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:36.323 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3777781316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4314153a-0a5d-44cb-b4fb-9e7e1c02118d"}]: dispatch 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3777781316' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4314153a-0a5d-44cb-b4fb-9e7e1c02118d"}]': finished 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: osdmap e6: 1 total, 0 up, 1 in 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:36.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:36 smithi069 ceph-mon[25054]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:37.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:37 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1025447311' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:37:37.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:37 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1025447311' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:37:38.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:38 smithi069 ceph-mon[25054]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:38.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:38 smithi135 ceph-mon[27511]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:39.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:39.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:37:39.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:39.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:39.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: Reconfiguring mon.smithi069 (unknown last config time)... 2024-06-11T13:37:39.663 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:37:39.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:37:39.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:39.664 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:39 smithi069 ceph-mon[25054]: Reconfiguring daemon mon.smithi069 on smithi069 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: Reconfiguring mon.smithi069 (unknown last config time)... 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:39 smithi135 ceph-mon[27511]: Reconfiguring daemon mon.smithi069 on smithi069 2024-06-11T13:37:40.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:40 smithi069 ceph-mon[25054]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:41.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:40 smithi135 ceph-mon[27511]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: Reconfiguring mgr.smithi069.iruzyw (unknown last config time)... 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: Reconfiguring daemon mgr.smithi069.iruzyw on smithi069 2024-06-11T13:37:42.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:41 smithi069 ceph-mon[25054]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:42.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:42.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: Reconfiguring mgr.smithi069.iruzyw (unknown last config time)... 2024-06-11T13:37:42.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:37:42.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:37:42.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:42.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: Reconfiguring daemon mgr.smithi069.iruzyw on smithi069 2024-06-11T13:37:42.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:41 smithi135 ceph-mon[27511]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:43.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-11T13:37:43.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:43.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:42 smithi069 ceph-mon[25054]: Deploying daemon osd.0 on smithi069 2024-06-11T13:37:43.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-11T13:37:43.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:43.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:42 smithi135 ceph-mon[27511]: Deploying daemon osd.0 on smithi069 2024-06-11T13:37:44.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:44 smithi069 ceph-mon[25054]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-06-11T13:37:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:44 smithi069 ceph-mon[25054]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-06-11T13:37:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:44 smithi069 ceph-mon[25054]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:44 smithi135 ceph-mon[27511]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-06-11T13:37:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:44 smithi135 ceph-mon[27511]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-06-11T13:37:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:44 smithi135 ceph-mon[27511]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:46.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:46 smithi069 ceph-mon[25054]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:46.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:46 smithi135 ceph-mon[27511]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:48.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:48 smithi135 ceph-mon[27511]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:48.333 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:48 smithi069 ceph-mon[25054]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:49.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:49 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:49.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:49 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:50 smithi069 ceph-mon[25054]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:37:50.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:37:51.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:50 smithi135 ceph-mon[27511]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:37:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:37:52.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:51 smithi069 ceph-mon[25054]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-11T13:37:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:51 smithi135 ceph-mon[27511]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-11T13:37:52.992 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:52 smithi069 ceph-mon[25054]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-11T13:37:52.992 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:52 smithi069 ceph-mon[25054]: osdmap e7: 1 total, 0 up, 1 in 2024-06-11T13:37:52.992 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:52.992 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:52 smithi069 ceph-mon[25054]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:37:52.992 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:52 smithi069 ceph-mon[25054]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:52 smithi135 ceph-mon[27511]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-11T13:37:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:52 smithi135 ceph-mon[27511]: osdmap e7: 1 total, 0 up, 1 in 2024-06-11T13:37:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:52 smithi135 ceph-mon[27511]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:37:53.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:52 smithi135 ceph-mon[27511]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:54.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:37:54.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: osdmap e8: 1 total, 0 up, 1 in 2024-06-11T13:37:54.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:54.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:54.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:54.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:37:54.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:54.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:54.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: osdmap e8: 1 total, 0 up, 1 in 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:37:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: Reconfiguring crash.smithi069 (monmap changed)... 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: Reconfiguring daemon crash.smithi069 on smithi069 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411] boot 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: osdmap e9: 1 total, 1 up, 1 in 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:55.147 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:54 smithi069 ceph-mon[25054]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: Reconfiguring crash.smithi069 (monmap changed)... 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: Reconfiguring daemon crash.smithi069 on smithi069 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: osd.0 [v2:172.21.15.69:6802/33340411,v1:172.21.15.69:6803/33340411] boot 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: osdmap e9: 1 total, 1 up, 1 in 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:37:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:54 smithi135 ceph-mon[27511]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-06-11T13:37:55.460 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 0 on host 'smithi069' 2024-06-11T13:37:55.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:55 smithi069 ceph-mon[25054]: osdmap e10: 1 total, 1 up, 1 in 2024-06-11T13:37:55.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:56.268 DEBUG:teuthology.orchestra.run.smithi069:osd.0> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.0.service 2024-06-11T13:37:56.271 INFO:tasks.cephadm:Deploying osd.1 on smithi069 with /dev/vg_nvme/lv_3... 2024-06-11T13:37:56.271 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-11T13:37:56.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:55 smithi135 ceph-mon[27511]: osdmap e10: 1 total, 1 up, 1 in 2024-06-11T13:37:56.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:57.098 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:56 smithi069 ceph-mon[25054]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:37:57.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:56 smithi135 ceph-mon[27511]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:37:58.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:58.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:58 smithi069 ceph-mon[25054]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-06-11T13:37:58.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:58 smithi069 ceph-mon[25054]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-06-11T13:37:58.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:37:58 smithi069 ceph-mon[25054]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:37:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:58 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:37:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:58 smithi135 ceph-mon[27511]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-06-11T13:37:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:58 smithi135 ceph-mon[27511]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-06-11T13:37:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:37:58 smithi135 ceph-mon[27511]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:37:59.739 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-11T13:37:59.739 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-11T13:37:59.739 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-06-11T13:37:59.739 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-06-11T13:37:59.739 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0176453 s, 594 MB/s 2024-06-11T13:37:59.739 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-06-11T13:38:00.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:00 smithi069 ceph-mon[25054]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:00.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:00 smithi135 ceph-mon[27511]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:01.686 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_3 2024-06-11T13:38:02.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:02 smithi135 ceph-mon[27511]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:02.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:02 smithi069 ceph-mon[25054]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:04 smithi069 ceph-mon[25054]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:04.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:04 smithi135 ceph-mon[27511]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='client.24117 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: Reconfiguring crash.smithi135 (monmap changed)... 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:05 smithi135 ceph-mon[27511]: Reconfiguring daemon crash.smithi135 on smithi135 2024-06-11T13:38:05.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='client.24117 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:38:05.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: Reconfiguring crash.smithi135 (monmap changed)... 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:05 smithi069 ceph-mon[25054]: Reconfiguring daemon crash.smithi135 on smithi135 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: Reconfiguring mgr.smithi135.xcould (monmap changed)... 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: Reconfiguring daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:06 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3501442288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d24d902f-6d71-4b9c-83d9-88cd91adae00"}]: dispatch 2024-06-11T13:38:07.021 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:07.021 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: Reconfiguring mgr.smithi135.xcould (monmap changed)... 2024-06-11T13:38:07.022 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:38:07.022 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:38:07.022 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:07.022 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: Reconfiguring daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:38:07.022 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:07.022 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:06 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3501442288' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d24d902f-6d71-4b9c-83d9-88cd91adae00"}]: dispatch 2024-06-11T13:38:07.673 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3501442288' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d24d902f-6d71-4b9c-83d9-88cd91adae00"}]': finished 2024-06-11T13:38:07.673 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: osdmap e11: 2 total, 1 up, 2 in 2024-06-11T13:38:07.674 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:07.674 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:07.674 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:38:07.674 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:38:07.674 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:07.674 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:07 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2578226461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:38:07.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3501442288' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d24d902f-6d71-4b9c-83d9-88cd91adae00"}]': finished 2024-06-11T13:38:07.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: osdmap e11: 2 total, 1 up, 2 in 2024-06-11T13:38:07.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:07.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:38:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:38:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:07 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2578226461' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:38:08.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:08 smithi069 ceph-mon[25054]: Reconfiguring mon.smithi135 (monmap changed)... 2024-06-11T13:38:08.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:08 smithi069 ceph-mon[25054]: Reconfiguring daemon mon.smithi135 on smithi135 2024-06-11T13:38:08.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:08 smithi069 ceph-mon[25054]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:08.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:08.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:38:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:08 smithi135 ceph-mon[27511]: Reconfiguring mon.smithi135 (monmap changed)... 2024-06-11T13:38:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:08 smithi135 ceph-mon[27511]: Reconfiguring daemon mon.smithi135 on smithi135 2024-06-11T13:38:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:08 smithi135 ceph-mon[27511]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:38:09.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:09 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:38:09.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:09 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:38:10.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:10 smithi069 ceph-mon[25054]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:11.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:10 smithi135 ceph-mon[27511]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:12.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:12 smithi135 ceph-mon[27511]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:12.371 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:12 smithi069 ceph-mon[25054]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:13.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-11T13:38:13.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:13.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:13 smithi135 ceph-mon[27511]: Deploying daemon osd.1 on smithi069 2024-06-11T13:38:13.307 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-11T13:38:13.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:13.308 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:13 smithi069 ceph-mon[25054]: Deploying daemon osd.1 on smithi069 2024-06-11T13:38:14.178 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:14 smithi069 ceph-mon[25054]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:14.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:14 smithi135 ceph-mon[27511]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:16.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:16 smithi069 ceph-mon[25054]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:16.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:16 smithi135 ceph-mon[27511]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:18.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:18 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:18.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:18 smithi069 ceph-mon[25054]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:19.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:18 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:19.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:18 smithi135 ceph-mon[27511]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:20.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:20 smithi135 ceph-mon[27511]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:20.490 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:20 smithi069 ceph-mon[25054]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:20.521 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 1 on host 'smithi069' 2024-06-11T13:38:21.130 DEBUG:teuthology.orchestra.run.smithi069:osd.1> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.1.service 2024-06-11T13:38:21.132 INFO:tasks.cephadm:Deploying osd.2 on smithi069 with /dev/vg_nvme/lv_2... 2024-06-11T13:38:21.132 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-11T13:38:21.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-06-11T13:38:21.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-06-11T13:38:21.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:21.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:21.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:21.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:21 smithi069 ceph-mon[25054]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.69:9093"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.69:3000"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:38:21.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.69:9095"}]: dispatch 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:21 smithi135 ceph-mon[27511]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-11T13:38:22.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:22 smithi135 ceph-mon[27511]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-11T13:38:22.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:22 smithi135 ceph-mon[27511]: osdmap e12: 2 total, 1 up, 2 in 2024-06-11T13:38:22.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:22.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:22 smithi135 ceph-mon[27511]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:38:22.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:22 smithi135 ceph-mon[27511]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:22.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:22.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-mon[25054]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-11T13:38:22.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-mon[25054]: osdmap e12: 2 total, 1 up, 2 in 2024-06-11T13:38:22.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:22.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-mon[25054]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:38:22.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-mon[25054]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:22.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:22.983 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T13:38:22.560+0000 7fa8435e2700 -1 osd.1 0 waiting for initial osdmap 2024-06-11T13:38:22.983 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:22 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T13:38:22.562+0000 7fa83cf77700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:38:23.745 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:23 smithi069 ceph-mon[25054]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:38:23.745 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:23 smithi069 ceph-mon[25054]: osdmap e13: 2 total, 1 up, 2 in 2024-06-11T13:38:23.745 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:23.745 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:23 smithi135 ceph-mon[27511]: from='osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:38:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:23 smithi135 ceph-mon[27511]: osdmap e13: 2 total, 1 up, 2 in 2024-06-11T13:38:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:24.409 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-11T13:38:24.409 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-11T13:38:24.409 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-06-11T13:38:24.409 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-06-11T13:38:24.409 INFO:teuthology.orchestra.run.smithi069.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0149607 s, 701 MB/s 2024-06-11T13:38:24.409 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-06-11T13:38:24.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:24 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:38:24.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:24 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:38:24.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:24 smithi069 ceph-mon[25054]: osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180] boot 2024-06-11T13:38:24.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:24 smithi069 ceph-mon[25054]: osdmap e14: 2 total, 2 up, 2 in 2024-06-11T13:38:24.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:24 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:24.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:24 smithi069 ceph-mon[25054]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:25.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:24 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:38:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:24 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:38:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:24 smithi135 ceph-mon[27511]: osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180] boot 2024-06-11T13:38:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:24 smithi135 ceph-mon[27511]: osdmap e14: 2 total, 2 up, 2 in 2024-06-11T13:38:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:24 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:38:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:24 smithi135 ceph-mon[27511]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-06-11T13:38:25.332 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_2 2024-06-11T13:38:25.868 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:25 smithi069 ceph-mon[25054]: osdmap e15: 2 total, 2 up, 2 in 2024-06-11T13:38:26.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:25 smithi135 ceph-mon[27511]: osdmap e15: 2 total, 2 up, 2 in 2024-06-11T13:38:26.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:26 smithi069 ceph-mon[25054]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:27.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:26 smithi135 ceph-mon[27511]: pgmap v51: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:29.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:29.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:28 smithi069 ceph-mon[25054]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:29.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:38:29.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:38:29.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:29.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:29.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:28 smithi135 ceph-mon[27511]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:29.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:38:29.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:38:29.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:30.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:29 smithi069 ceph-mon[25054]: from='client.14276 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:38:30.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:29 smithi135 ceph-mon[27511]: from='client.14276 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:38:30.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:30 smithi069 ceph-mon[25054]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:31.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:30 smithi135 ceph-mon[27511]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:32.140 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:32 smithi069 ceph-mon[25054]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:32.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:32 smithi135 ceph-mon[27511]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi069 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi069 to 6432M 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:33.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:33.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi069 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi069 to 6432M 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:33.769 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:33.770 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:33.770 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:33.770 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:34.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1008395941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "496987fe-a059-4050-9f86-df5ba85e744c"}]: dispatch 2024-06-11T13:38:34.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1008395941' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "496987fe-a059-4050-9f86-df5ba85e744c"}]': finished 2024-06-11T13:38:34.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: osdmap e16: 3 total, 2 up, 3 in 2024-06-11T13:38:34.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:34.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:34.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:34.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:34 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/702868581' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1008395941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "496987fe-a059-4050-9f86-df5ba85e744c"}]: dispatch 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1008395941' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "496987fe-a059-4050-9f86-df5ba85e744c"}]': finished 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: osdmap e16: 3 total, 2 up, 3 in 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:34.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:34 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/702868581' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:38:36.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:36 smithi069 ceph-mon[25054]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:36.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:36 smithi135 ceph-mon[27511]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:37.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:37 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:38.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:37 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:38.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:38 smithi069 ceph-mon[25054]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:39.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:38 smithi135 ceph-mon[27511]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:40.162 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:40 smithi069 ceph-mon[25054]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:40.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:40 smithi135 ceph-mon[27511]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:41.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:41 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi069 2024-06-11T13:38:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:41 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:41 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:41.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:41 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi069 2024-06-11T13:38:41.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:41 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:41.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:41 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:42.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-11T13:38:42.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:42.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:42 smithi069 ceph-mon[25054]: Deploying daemon osd.2 on smithi069 2024-06-11T13:38:42.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:42 smithi069 ceph-mon[25054]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:42.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-11T13:38:42.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:42.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:42 smithi135 ceph-mon[27511]: Deploying daemon osd.2 on smithi069 2024-06-11T13:38:42.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:42 smithi135 ceph-mon[27511]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:44.150 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:44 smithi069 ceph-mon[25054]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:44.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:44 smithi135 ceph-mon[27511]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:46.607 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:46 smithi069 ceph-mon[25054]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:46.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:46 smithi135 ceph-mon[27511]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:48.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:48.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:48.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:48.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:48.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:48.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:48.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:48.191 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:49.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:48 smithi069 ceph-mon[25054]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:49.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:49.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:48 smithi135 ceph-mon[27511]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:49.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:49.858 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 2 on host 'smithi069' 2024-06-11T13:38:51.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:51.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:50 smithi069 ceph-mon[25054]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-11T13:38:51.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:50 smithi069 ceph-mon[25054]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:51.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:38:51.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:38:51.114 DEBUG:teuthology.orchestra.run.smithi069:osd.2> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.2.service 2024-06-11T13:38:51.117 INFO:tasks.cephadm:Deploying osd.3 on smithi069 with /dev/vg_nvme/lv_1... 2024-06-11T13:38:51.117 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-11T13:38:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:50 smithi135 ceph-mon[27511]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-11T13:38:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:50 smithi135 ceph-mon[27511]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:38:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:38:51.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:51 smithi069 ceph-mon[25054]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-11T13:38:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:51 smithi069 ceph-mon[25054]: osdmap e17: 3 total, 2 up, 3 in 2024-06-11T13:38:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:51 smithi069 ceph-mon[25054]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:38:51.997 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:51 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T13:38:51.867+0000 7fc30b1f8700 -1 osd.2 0 waiting for initial osdmap 2024-06-11T13:38:51.997 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:51 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T13:38:51.871+0000 7fc305b8f700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:38:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:51 smithi135 ceph-mon[27511]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-11T13:38:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:51 smithi135 ceph-mon[27511]: osdmap e17: 3 total, 2 up, 3 in 2024-06-11T13:38:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:51 smithi135 ceph-mon[27511]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:38:53.185 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:52 smithi069 ceph-mon[25054]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:38:53.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:52 smithi069 ceph-mon[25054]: osdmap e18: 3 total, 2 up, 3 in 2024-06-11T13:38:53.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:53.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:53.186 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:52 smithi069 ceph-mon[25054]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:53.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:52 smithi135 ceph-mon[27511]: from='osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:38:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:52 smithi135 ceph-mon[27511]: osdmap e18: 3 total, 2 up, 3 in 2024-06-11T13:38:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:52 smithi135 ceph-mon[27511]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-06-11T13:38:54.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:53 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:38:54.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:53 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:38:54.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:54.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:53 smithi069 ceph-mon[25054]: osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448] boot 2024-06-11T13:38:54.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:53 smithi069 ceph-mon[25054]: osdmap e19: 3 total, 3 up, 3 in 2024-06-11T13:38:54.136 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:53 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:38:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:53 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:38:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:53 smithi135 ceph-mon[27511]: osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448] boot 2024-06-11T13:38:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:53 smithi135 ceph-mon[27511]: osdmap e19: 3 total, 3 up, 3 in 2024-06-11T13:38:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:38:54.795 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-11T13:38:54.795 INFO:teuthology.orchestra.run.smithi069.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-11T13:38:54.795 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10+0 records in 2024-06-11T13:38:54.795 INFO:teuthology.orchestra.run.smithi069.stderr:10+0 records out 2024-06-11T13:38:54.795 INFO:teuthology.orchestra.run.smithi069.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0139965 s, 749 MB/s 2024-06-11T13:38:54.795 INFO:teuthology.orchestra.run.smithi069.stderr:--> Zapping successful for: 2024-06-11T13:38:54.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:54 smithi069 ceph-mon[25054]: osdmap e20: 3 total, 3 up, 3 in 2024-06-11T13:38:54.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:54 smithi069 ceph-mon[25054]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:38:54.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:54 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-11T13:38:54.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:54 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:55.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:54 smithi135 ceph-mon[27511]: osdmap e20: 3 total, 3 up, 3 in 2024-06-11T13:38:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:54 smithi135 ceph-mon[27511]: pgmap v70: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:38:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:54 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-06-11T13:38:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:54 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:55.465 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi069:vg_nvme/lv_1 2024-06-11T13:38:56.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-11T13:38:56.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:55 smithi069 ceph-mon[25054]: osdmap e21: 3 total, 3 up, 3 in 2024-06-11T13:38:56.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-11T13:38:56.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-06-11T13:38:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:55 smithi135 ceph-mon[27511]: osdmap e21: 3 total, 3 up, 3 in 2024-06-11T13:38:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-06-11T13:38:56.598 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:38:56 smithi069 sudo[43617]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-11T13:38:56.599 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:38:56 smithi069 sudo[43617]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:56.599 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:38:56 smithi069 sudo[43617]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:56.860 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:38:56 smithi069 sudo[43641]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-11T13:38:56.860 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:38:56 smithi069 sudo[43641]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:56.860 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:38:56 smithi069 sudo[43641]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:57.112 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43648]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-11T13:38:57.112 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43648]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:57.112 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43648]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:57.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:56 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-11T13:38:57.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:56 smithi069 ceph-mon[25054]: osdmap e22: 3 total, 3 up, 3 in 2024-06-11T13:38:57.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:56 smithi069 ceph-mon[25054]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:38:57.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:56 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:57.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:56 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-06-11T13:38:57.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:56 smithi135 ceph-mon[27511]: osdmap e22: 3 total, 3 up, 3 in 2024-06-11T13:38:57.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:56 smithi135 ceph-mon[27511]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:38:57.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:56 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:57.496 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43669]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-11T13:38:57.496 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43669]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:57.496 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43669]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:57.829 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43673]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-11T13:38:57.829 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43673]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:57.830 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43673]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:58.080 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43695]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-11T13:38:58.080 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43695]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:58.080 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:38:57 smithi069 sudo[43695]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:58.348 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 sudo[43701]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-11T13:38:58.348 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 sudo[43701]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:58.348 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 sudo[43701]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:58.609 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 ceph-mon[27511]: osdmap e23: 3 total, 3 up, 3 in 2024-06-11T13:38:58.609 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 sudo[29798]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-06-11T13:38:58.609 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 sudo[29798]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:58.609 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 sudo[29798]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:58.618 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 sudo[43728]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-11T13:38:58.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 sudo[43728]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:58.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 sudo[43728]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:58.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:58 smithi069 ceph-mon[25054]: osdmap e23: 3 total, 3 up, 3 in 2024-06-11T13:38:59.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 sudo[29801]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-06-11T13:38:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 sudo[29801]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-06-11T13:38:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:58 smithi135 sudo[29801]: pam_unix(sudo:session): session closed for user root 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='client.14300 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:38:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: mgrmap e19: smithi069.iruzyw(active, since 2m), standbys: smithi135.xcould 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:59.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:59.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:38:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:38:59.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-11T13:38:59.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:38:59.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-11T13:38:59.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:38:59.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:38:59.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='client.14300 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi069:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: mgrmap e19: smithi069.iruzyw(active, since 2m), standbys: smithi135.xcould 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:38:59.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:38:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:00.406 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:00 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi069 to 4288M 2024-06-11T13:39:00.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:00 smithi069 ceph-mon[25054]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:00.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:00 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi069 to 4288M 2024-06-11T13:39:00.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:00 smithi135 ceph-mon[27511]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:01.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:01 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/379282918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "10b22a7a-0b19-4284-b8e1-276e1e80441f"}]: dispatch 2024-06-11T13:39:01.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:01 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/379282918' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "10b22a7a-0b19-4284-b8e1-276e1e80441f"}]': finished 2024-06-11T13:39:01.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:01 smithi135 ceph-mon[27511]: osdmap e24: 4 total, 3 up, 4 in 2024-06-11T13:39:01.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:01 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:01.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:01 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/379282918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "10b22a7a-0b19-4284-b8e1-276e1e80441f"}]: dispatch 2024-06-11T13:39:01.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:01 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/379282918' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "10b22a7a-0b19-4284-b8e1-276e1e80441f"}]': finished 2024-06-11T13:39:01.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:01 smithi069 ceph-mon[25054]: osdmap e24: 4 total, 3 up, 4 in 2024-06-11T13:39:01.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:01 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:02.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:02 smithi135 ceph-mon[27511]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:02.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:02 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1321072018' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:39:02.828 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:02 smithi069 ceph-mon[25054]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:02.828 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:02 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1321072018' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:39:04.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:04 smithi135 ceph-mon[27511]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:04.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:04 smithi069 ceph-mon[25054]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:05.903 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:05.903 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:06.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:06.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:06.976 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:06 smithi069 ceph-mon[25054]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:06 smithi135 ceph-mon[27511]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:08.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-11T13:39:08.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:08.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:08 smithi069 ceph-mon[25054]: Deploying daemon osd.3 on smithi069 2024-06-11T13:39:08.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:08 smithi069 ceph-mon[25054]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:08.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-11T13:39:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:08 smithi135 ceph-mon[27511]: Deploying daemon osd.3 on smithi069 2024-06-11T13:39:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:08 smithi135 ceph-mon[27511]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:10.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:10 smithi069 ceph-mon[25054]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:10.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:10 smithi135 ceph-mon[27511]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:12.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:12 smithi069 ceph-mon[25054]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:12.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:12 smithi135 ceph-mon[27511]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:14.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:14.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:14.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:14.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:14.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:13 smithi069 ceph-mon[25054]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:14.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:13 smithi135 ceph-mon[27511]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:15.936 INFO:teuthology.orchestra.run.smithi069.stdout:Created osd(s) 3 on host 'smithi069' 2024-06-11T13:39:16.575 DEBUG:teuthology.orchestra.run.smithi069:osd.3> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.3.service 2024-06-11T13:39:16.577 INFO:tasks.cephadm:Deploying osd.4 on smithi135 with /dev/vg_nvme/lv_4... 2024-06-11T13:39:16.578 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-06-11T13:39:17.184 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:16 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:17.184 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:16 smithi135 ceph-mon[27511]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:17.184 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:16 smithi135 ceph-mon[27511]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-11T13:39:17.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:16 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:17.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:16 smithi069 ceph-mon[25054]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:17.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:16 smithi069 ceph-mon[25054]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-11T13:39:18.127 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:17 smithi135 ceph-mon[27511]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-11T13:39:18.128 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:17 smithi135 ceph-mon[27511]: osdmap e25: 4 total, 3 up, 4 in 2024-06-11T13:39:18.128 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:17 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:18.128 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:17 smithi135 ceph-mon[27511]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:39:18.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:17 smithi069 ceph-mon[25054]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-11T13:39:18.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:17 smithi069 ceph-mon[25054]: osdmap e25: 4 total, 3 up, 4 in 2024-06-11T13:39:18.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:17 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:18.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:17 smithi069 ceph-mon[25054]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T13:39:18.247 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:39:17 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:39:17.949+0000 7f596496a700 -1 osd.3 0 waiting for initial osdmap 2024-06-11T13:39:18.247 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:39:17 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:39:17.958+0000 7f595d2fd700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:39:18.599 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-06-11T13:39:18.599 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-06-11T13:39:18.599 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2024-06-11T13:39:18.599 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2024-06-11T13:39:18.599 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128408 s, 817 MB/s 2024-06-11T13:39:18.600 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2024-06-11T13:39:19.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:18 smithi069 ceph-mon[25054]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:39:19.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:18 smithi069 ceph-mon[25054]: osdmap e26: 4 total, 3 up, 4 in 2024-06-11T13:39:19.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:18 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:19.125 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:18 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:19.125 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:18 smithi069 ceph-mon[25054]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:19.158 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi135:vg_nvme/lv_4 2024-06-11T13:39:19.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:18 smithi135 ceph-mon[27511]: from='osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]': finished 2024-06-11T13:39:19.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:18 smithi135 ceph-mon[27511]: osdmap e26: 4 total, 3 up, 4 in 2024-06-11T13:39:19.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:18 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:19.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:18 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:19.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:18 smithi135 ceph-mon[27511]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534] boot 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: osdmap e27: 4 total, 4 up, 4 in 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:19 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:20.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:39:20.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:39:20.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:20.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: osd.3 [v2:172.21.15.69:6826/1053767534,v1:172.21.15.69:6827/1053767534] boot 2024-06-11T13:39:20.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: osdmap e27: 4 total, 4 up, 4 in 2024-06-11T13:39:20.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:39:20.218 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:19 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:21.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:20 smithi135 ceph-mon[27511]: osdmap e28: 4 total, 4 up, 4 in 2024-06-11T13:39:21.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:20 smithi135 ceph-mon[27511]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:21.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:20 smithi069 ceph-mon[25054]: osdmap e28: 4 total, 4 up, 4 in 2024-06-11T13:39:21.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:20 smithi069 ceph-mon[25054]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:22.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:39:22.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:39:22.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:22.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:39:22.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:39:22.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:23.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:22 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:23.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:23.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:23.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:23.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:23.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:22 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:24.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi069 2024-06-11T13:39:24.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi069 to 3216M 2024-06-11T13:39:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/678016430' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c2eb3ce0-dac9-4447-97e9-ae8c1068503b"}]: dispatch 2024-06-11T13:39:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c2eb3ce0-dac9-4447-97e9-ae8c1068503b"}]: dispatch 2024-06-11T13:39:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c2eb3ce0-dac9-4447-97e9-ae8c1068503b"}]': finished 2024-06-11T13:39:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: osdmap e29: 5 total, 4 up, 5 in 2024-06-11T13:39:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:23 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2853191487' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi069 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi069 to 3216M 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/678016430' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c2eb3ce0-dac9-4447-97e9-ae8c1068503b"}]: dispatch 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c2eb3ce0-dac9-4447-97e9-ae8c1068503b"}]: dispatch 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c2eb3ce0-dac9-4447-97e9-ae8c1068503b"}]': finished 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: osdmap e29: 5 total, 4 up, 5 in 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:23 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2853191487' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:39:25.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:24 smithi069 ceph-mon[25054]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:25.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:24 smithi135 ceph-mon[27511]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:26.726 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:26 smithi135 ceph-mon[27511]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:26.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:26 smithi069 ceph-mon[25054]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:27.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:27 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:27.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:27 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:27.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:27 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:27.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:27 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:28.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:28 smithi069 ceph-mon[25054]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:29.031 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:28 smithi135 ceph-mon[27511]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:29.877 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-11T13:39:29.877 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:29.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-11T13:39:29.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:30.880 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:30 smithi135 ceph-mon[27511]: Deploying daemon osd.4 on smithi135 2024-06-11T13:39:30.880 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:30 smithi135 ceph-mon[27511]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:30.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:30 smithi069 ceph-mon[25054]: Deploying daemon osd.4 on smithi135 2024-06-11T13:39:30.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:30 smithi069 ceph-mon[25054]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:32.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:32 smithi135 ceph-mon[27511]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:32.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:32 smithi069 ceph-mon[25054]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:34.295 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:34 smithi135 ceph-mon[27511]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:34.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:34 smithi069 ceph-mon[25054]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:35.440 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:35.440 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:35.440 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:35.440 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:36.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:36 smithi135 ceph-mon[27511]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:36.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:36 smithi069 ceph-mon[25054]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:37.593 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 4 on host 'smithi135' 2024-06-11T13:39:37.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:37 smithi135 ceph-mon[27511]: from='osd.4 [v2:172.21.15.135:6800/90669583,v1:172.21.15.135:6801/90669583]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T13:39:37.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:37 smithi135 ceph-mon[27511]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T13:39:37.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:37 smithi069 ceph-mon[25054]: from='osd.4 [v2:172.21.15.135:6800/90669583,v1:172.21.15.135:6801/90669583]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T13:39:37.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:37 smithi069 ceph-mon[25054]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T13:39:38.090 DEBUG:teuthology.orchestra.run.smithi135:osd.4> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.4.service 2024-06-11T13:39:38.093 INFO:tasks.cephadm:Deploying osd.5 on smithi135 with /dev/vg_nvme/lv_3... 2024-06-11T13:39:38.093 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-06-11T13:39:38.772 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-11T13:39:38.772 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: osdmap e30: 5 total, 4 up, 5 in 2024-06-11T13:39:38.772 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:38.772 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: from='osd.4 [v2:172.21.15.135:6800/90669583,v1:172.21.15.135:6801/90669583]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:39:38.773 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:39:38.773 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:38.773 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:38.773 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:38.773 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T13:39:38.533+0000 7f9441735700 -1 osd.4 0 waiting for initial osdmap 2024-06-11T13:39:38.773 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 13:39:38 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T13:39:38.537+0000 7f943b8cb700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:39:38.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-11T13:39:38.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: osdmap e30: 5 total, 4 up, 5 in 2024-06-11T13:39:38.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: from='osd.4 [v2:172.21.15.135:6800/90669583,v1:172.21.15.135:6801/90669583]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:39:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:39:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:38 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:39 smithi135 ceph-mon[27511]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:39:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:39 smithi135 ceph-mon[27511]: osdmap e31: 5 total, 4 up, 5 in 2024-06-11T13:39:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:39.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:39 smithi069 ceph-mon[25054]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:39:39.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:39 smithi069 ceph-mon[25054]: osdmap e31: 5 total, 4 up, 5 in 2024-06-11T13:39:39.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:39.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: osd.4 [v2:172.21.15.135:6800/90669583,v1:172.21.15.135:6801/90669583] boot 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: osdmap e32: 5 total, 5 up, 5 in 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: pgmap v105: 1 pgs: 1 remapped+peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:40 smithi135 ceph-mon[27511]: osdmap e33: 5 total, 5 up, 5 in 2024-06-11T13:39:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:39:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:39:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: osd.4 [v2:172.21.15.135:6800/90669583,v1:172.21.15.135:6801/90669583] boot 2024-06-11T13:39:40.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: osdmap e32: 5 total, 5 up, 5 in 2024-06-11T13:39:40.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:39:40.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: pgmap v105: 1 pgs: 1 remapped+peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-06-11T13:39:40.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:40 smithi069 ceph-mon[25054]: osdmap e33: 5 total, 5 up, 5 in 2024-06-11T13:39:41.537 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-06-11T13:39:41.537 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-06-11T13:39:41.537 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2024-06-11T13:39:41.537 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2024-06-11T13:39:41.537 INFO:teuthology.orchestra.run.smithi135.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125737 s, 834 MB/s 2024-06-11T13:39:41.537 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2024-06-11T13:39:42.184 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi135:vg_nvme/lv_3 2024-06-11T13:39:42.764 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:42 smithi135 ceph-mon[27511]: osdmap e34: 5 total, 5 up, 5 in 2024-06-11T13:39:42.764 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:42 smithi135 ceph-mon[27511]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:39:42.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:42 smithi069 ceph-mon[25054]: osdmap e34: 5 total, 5 up, 5 in 2024-06-11T13:39:42.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:42 smithi069 ceph-mon[25054]: pgmap v108: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: osdmap e35: 5 total, 5 up, 5 in 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi135 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi135 to 15936M 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:43.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:43 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: osdmap e35: 5 total, 5 up, 5 in 2024-06-11T13:39:43.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi135 2024-06-11T13:39:43.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi135 to 15936M 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:43.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:43 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:44.789 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:44 smithi135 ceph-mon[27511]: osdmap e36: 5 total, 5 up, 5 in 2024-06-11T13:39:44.790 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:44 smithi135 ceph-mon[27511]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:39:44.790 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:39:44.790 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:39:44.791 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:44.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:44 smithi069 ceph-mon[25054]: osdmap e36: 5 total, 5 up, 5 in 2024-06-11T13:39:44.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:44 smithi069 ceph-mon[25054]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:39:44.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:39:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:39:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:45.754 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:45 smithi135 ceph-mon[27511]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:39:45.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:45 smithi069 ceph-mon[25054]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:39:46.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:46 smithi135 ceph-mon[27511]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-06-11T13:39:46.898 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:46 smithi069 ceph-mon[25054]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-06-11T13:39:47.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:47 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/1810203408' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a8e63d3-9163-464c-86a6-4ceb5ed183a5"}]: dispatch 2024-06-11T13:39:47.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:47 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a8e63d3-9163-464c-86a6-4ceb5ed183a5"}]: dispatch 2024-06-11T13:39:47.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:47 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a8e63d3-9163-464c-86a6-4ceb5ed183a5"}]': finished 2024-06-11T13:39:47.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:47 smithi135 ceph-mon[27511]: osdmap e37: 6 total, 5 up, 6 in 2024-06-11T13:39:47.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:39:47.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:47 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/1848862534' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:39:47.914 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:47 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/1810203408' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a8e63d3-9163-464c-86a6-4ceb5ed183a5"}]: dispatch 2024-06-11T13:39:47.914 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:47 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a8e63d3-9163-464c-86a6-4ceb5ed183a5"}]: dispatch 2024-06-11T13:39:47.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:47 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a8e63d3-9163-464c-86a6-4ceb5ed183a5"}]': finished 2024-06-11T13:39:47.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:47 smithi069 ceph-mon[25054]: osdmap e37: 6 total, 5 up, 6 in 2024-06-11T13:39:47.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:39:47.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:47 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/1848862534' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:39:48.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:48 smithi069 ceph-mon[25054]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-11T13:39:48.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:48 smithi135 ceph-mon[27511]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-11T13:39:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:50 smithi135 ceph-mon[27511]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-06-11T13:39:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:39:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:39:50.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:50 smithi069 ceph-mon[25054]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-06-11T13:39:50.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:39:50.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:39:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:53 smithi135 ceph-mon[27511]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-06-11T13:39:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-11T13:39:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:53 smithi135 ceph-mon[27511]: Deploying daemon osd.5 on smithi135 2024-06-11T13:39:53.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:53 smithi069 ceph-mon[25054]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-06-11T13:39:53.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-11T13:39:53.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:53.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:53 smithi069 ceph-mon[25054]: Deploying daemon osd.5 on smithi135 2024-06-11T13:39:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:55 smithi135 ceph-mon[27511]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:39:55.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:55 smithi069 ceph-mon[25054]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:39:57.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:57 smithi135 ceph-mon[27511]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:39:57.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:57 smithi069 ceph-mon[25054]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:39:59.125 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:59.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:59.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:59.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:58 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:39:59.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:58 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:39:59.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:58 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:39:59.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:58 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:39:59.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:58 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:00.203 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:39:59 smithi135 ceph-mon[27511]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:40:00.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:39:59 smithi069 ceph-mon[25054]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:40:01.013 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:00 smithi135 ceph-mon[27511]: overall HEALTH_OK 2024-06-11T13:40:01.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:00 smithi069 ceph-mon[25054]: overall HEALTH_OK 2024-06-11T13:40:01.414 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 5 on host 'smithi135' 2024-06-11T13:40:01.937 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:01 smithi135 ceph-mon[27511]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:40:01.937 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:01 smithi135 ceph-mon[27511]: from='osd.5 [v2:172.21.15.135:6808/3461261274,v1:172.21.15.135:6809/3461261274]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T13:40:01.937 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:01 smithi135 ceph-mon[27511]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T13:40:01.938 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:01 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:02.239 DEBUG:teuthology.orchestra.run.smithi135:osd.5> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.5.service 2024-06-11T13:40:02.242 INFO:tasks.cephadm:Deploying osd.6 on smithi135 with /dev/vg_nvme/lv_2... 2024-06-11T13:40:02.242 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-06-11T13:40:02.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:01 smithi069 ceph-mon[25054]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:40:02.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:01 smithi069 ceph-mon[25054]: from='osd.5 [v2:172.21.15.135:6808/3461261274,v1:172.21.15.135:6809/3461261274]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T13:40:02.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:01 smithi069 ceph-mon[25054]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T13:40:02.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:01 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:03.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:02 smithi069 ceph-mon[25054]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-11T13:40:03.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:02 smithi069 ceph-mon[25054]: osdmap e38: 6 total, 5 up, 6 in 2024-06-11T13:40:03.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:02 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:03.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:02 smithi069 ceph-mon[25054]: from='osd.5 [v2:172.21.15.135:6808/3461261274,v1:172.21.15.135:6809/3461261274]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:03.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:02 smithi069 ceph-mon[25054]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:03.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:02 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:03.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-mon[27511]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-11T13:40:03.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-mon[27511]: osdmap e38: 6 total, 5 up, 6 in 2024-06-11T13:40:03.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:03.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-mon[27511]: from='osd.5 [v2:172.21.15.135:6808/3461261274,v1:172.21.15.135:6809/3461261274]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:03.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-mon[27511]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:03.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:03.296 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T13:40:02.906+0000 7f85d6083700 -1 osd.5 0 waiting for initial osdmap 2024-06-11T13:40:03.297 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 13:40:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T13:40:02.911+0000 7f85cd213700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:40:04.191 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:03 smithi135 ceph-mon[27511]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:40:04.191 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:03 smithi135 ceph-mon[27511]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:40:04.191 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:03 smithi135 ceph-mon[27511]: osdmap e39: 6 total, 5 up, 6 in 2024-06-11T13:40:04.192 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:03 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:04.192 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:03 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:03 smithi069 ceph-mon[25054]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-06-11T13:40:04.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:03 smithi069 ceph-mon[25054]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:40:04.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:03 smithi069 ceph-mon[25054]: osdmap e39: 6 total, 5 up, 6 in 2024-06-11T13:40:04.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:04.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:03 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:04.996 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-06-11T13:40:04.996 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-06-11T13:40:04.996 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2024-06-11T13:40:04.996 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2024-06-11T13:40:04.996 INFO:teuthology.orchestra.run.smithi135.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012296 s, 853 MB/s 2024-06-11T13:40:04.997 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2024-06-11T13:40:05.237 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:04 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:40:05.237 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:04 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:40:05.238 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:04 smithi135 ceph-mon[27511]: osd.5 [v2:172.21.15.135:6808/3461261274,v1:172.21.15.135:6809/3461261274] boot 2024-06-11T13:40:05.238 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:04 smithi135 ceph-mon[27511]: osdmap e40: 6 total, 6 up, 6 in 2024-06-11T13:40:05.238 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:04 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:05.238 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:04 smithi135 ceph-mon[27511]: osdmap e41: 6 total, 6 up, 6 in 2024-06-11T13:40:05.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:04 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:40:05.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:04 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:40:05.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:04 smithi069 ceph-mon[25054]: osd.5 [v2:172.21.15.135:6808/3461261274,v1:172.21.15.135:6809/3461261274] boot 2024-06-11T13:40:05.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:04 smithi069 ceph-mon[25054]: osdmap e40: 6 total, 6 up, 6 in 2024-06-11T13:40:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:04 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:40:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:04 smithi069 ceph-mon[25054]: osdmap e41: 6 total, 6 up, 6 in 2024-06-11T13:40:06.110 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi135:vg_nvme/lv_2 2024-06-11T13:40:06.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:05 smithi135 ceph-mon[27511]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:06.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:05 smithi135 ceph-mon[27511]: osdmap e42: 6 total, 6 up, 6 in 2024-06-11T13:40:06.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:05 smithi069 ceph-mon[25054]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:06.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:05 smithi069 ceph-mon[25054]: osdmap e42: 6 total, 6 up, 6 in 2024-06-11T13:40:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:06 smithi135 ceph-mon[27511]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-11T13:40:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:06 smithi135 ceph-mon[27511]: osdmap e43: 6 total, 6 up, 6 in 2024-06-11T13:40:07.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:06 smithi069 ceph-mon[25054]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-06-11T13:40:07.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:06 smithi069 ceph-mon[25054]: osdmap e43: 6 total, 6 up, 6 in 2024-06-11T13:40:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:07.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:07.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:07.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:07 smithi069 ceph-mon[25054]: osdmap e44: 6 total, 6 up, 6 in 2024-06-11T13:40:08.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:08.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:08.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:08.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:08.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:08.233 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:08.233 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:08.233 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:08.233 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:08.233 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:08.233 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:07 smithi135 ceph-mon[27511]: osdmap e44: 6 total, 6 up, 6 in 2024-06-11T13:40:09.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:08 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi135 2024-06-11T13:40:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:08 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi135 to 7968M 2024-06-11T13:40:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:40:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:40:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:09.269 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:08 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi135 2024-06-11T13:40:09.269 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:08 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi135 to 7968M 2024-06-11T13:40:09.270 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:40:09.270 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:40:09.270 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:10.182 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:09 smithi135 ceph-mon[27511]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:10.182 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:09 smithi135 ceph-mon[27511]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:40:10.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:09 smithi069 ceph-mon[25054]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:10.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:09 smithi069 ceph-mon[25054]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 102 KiB/s, 0 objects/s recovering 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: Cluster is now healthy 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2494542781' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2743be9d-a84a-431f-8afe-6c99cd0b6b0c"}]: dispatch 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2743be9d-a84a-431f-8afe-6c99cd0b6b0c"}]: dispatch 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2743be9d-a84a-431f-8afe-6c99cd0b6b0c"}]': finished 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: osdmap e45: 7 total, 6 up, 7 in 2024-06-11T13:40:11.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:11.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 102 KiB/s, 0 objects/s recovering 2024-06-11T13:40:11.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-06-11T13:40:11.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: Cluster is now healthy 2024-06-11T13:40:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2494542781' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2743be9d-a84a-431f-8afe-6c99cd0b6b0c"}]: dispatch 2024-06-11T13:40:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2743be9d-a84a-431f-8afe-6c99cd0b6b0c"}]: dispatch 2024-06-11T13:40:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2743be9d-a84a-431f-8afe-6c99cd0b6b0c"}]': finished 2024-06-11T13:40:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: osdmap e45: 7 total, 6 up, 7 in 2024-06-11T13:40:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:11 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:12.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:12 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/4008115120' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:40:12.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:12 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/4008115120' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:40:13.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:13 smithi135 ceph-mon[27511]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-11T13:40:13.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:13 smithi069 ceph-mon[25054]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-11T13:40:15.745 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:15 smithi135 ceph-mon[27511]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-11T13:40:15.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:15 smithi069 ceph-mon[25054]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-11T13:40:17.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:17 smithi135 ceph-mon[27511]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-06-11T13:40:17.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:17 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-11T13:40:17.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:17 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:17.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:17 smithi069 ceph-mon[25054]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2024-06-11T13:40:17.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:17 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-11T13:40:17.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:17 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:18.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:18 smithi135 ceph-mon[27511]: Deploying daemon osd.6 on smithi135 2024-06-11T13:40:18.946 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:18 smithi069 ceph-mon[25054]: Deploying daemon osd.6 on smithi135 2024-06-11T13:40:19.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:19 smithi135 ceph-mon[27511]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:40:19.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:19 smithi069 ceph-mon[25054]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:40:21.697 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:21 smithi135 ceph-mon[27511]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:21.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:21 smithi069 ceph-mon[25054]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:23.640 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:23 smithi135 ceph-mon[27511]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:23.640 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:23.640 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:23.640 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:23.640 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:23.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:23 smithi069 ceph-mon[25054]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:23.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:23.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:23.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:23.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:25.416 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:25 smithi135 ceph-mon[27511]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:25.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:25 smithi069 ceph-mon[25054]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:26.167 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 6 on host 'smithi135' 2024-06-11T13:40:26.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:26 smithi069 ceph-mon[25054]: from='osd.6 [v2:172.21.15.135:6816/1667247705,v1:172.21.15.135:6817/1667247705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T13:40:26.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:26 smithi069 ceph-mon[25054]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T13:40:26.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:26 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:26.795 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:26 smithi135 ceph-mon[27511]: from='osd.6 [v2:172.21.15.135:6816/1667247705,v1:172.21.15.135:6817/1667247705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T13:40:26.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:26 smithi135 ceph-mon[27511]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T13:40:26.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:26 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:26.940 DEBUG:teuthology.orchestra.run.smithi135:osd.6> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.6.service 2024-06-11T13:40:26.983 INFO:tasks.cephadm:Deploying osd.7 on smithi135 with /dev/vg_nvme/lv_1... 2024-06-11T13:40:26.983 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: osdmap e46: 7 total, 6 up, 7 in 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: from='osd.6 [v2:172.21.15.135:6816/1667247705,v1:172.21.15.135:6817/1667247705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:27.594 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:27.595 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[37639]: 2024-06-11T13:40:27.336+0000 7eff204c0700 -1 osd.6 0 waiting for initial osdmap 2024-06-11T13:40:27.595 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 13:40:27 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[37639]: 2024-06-11T13:40:27.343+0000 7eff1ce5b700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: osdmap e46: 7 total, 6 up, 7 in 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: from='osd.6 [v2:172.21.15.135:6816/1667247705,v1:172.21.15.135:6817/1667247705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:27.743 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:27 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:28.545 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:28 smithi135 ceph-mon[27511]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:40:28.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:28 smithi135 ceph-mon[27511]: osdmap e47: 7 total, 6 up, 7 in 2024-06-11T13:40:28.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:28.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:28.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:28 smithi069 ceph-mon[25054]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:40:28.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:28 smithi069 ceph-mon[25054]: osdmap e47: 7 total, 6 up, 7 in 2024-06-11T13:40:28.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:28.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:29.596 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:40:29.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:40:29.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:29.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: osd.6 [v2:172.21.15.135:6816/1667247705,v1:172.21.15.135:6817/1667247705] boot 2024-06-11T13:40:29.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: osdmap e48: 7 total, 7 up, 7 in 2024-06-11T13:40:29.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:29.597 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:29 smithi135 ceph-mon[27511]: osdmap e49: 7 total, 7 up, 7 in 2024-06-11T13:40:29.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:40:29.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:40:29.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-06-11T13:40:29.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: osd.6 [v2:172.21.15.135:6816/1667247705,v1:172.21.15.135:6817/1667247705] boot 2024-06-11T13:40:29.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: osdmap e48: 7 total, 7 up, 7 in 2024-06-11T13:40:29.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:40:29.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:29 smithi069 ceph-mon[25054]: osdmap e49: 7 total, 7 up, 7 in 2024-06-11T13:40:30.525 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-06-11T13:40:30.525 INFO:teuthology.orchestra.run.smithi135.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-06-11T13:40:30.525 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10+0 records in 2024-06-11T13:40:30.525 INFO:teuthology.orchestra.run.smithi135.stderr:10+0 records out 2024-06-11T13:40:30.526 INFO:teuthology.orchestra.run.smithi135.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112849 s, 929 MB/s 2024-06-11T13:40:30.526 INFO:teuthology.orchestra.run.smithi135.stderr:--> Zapping successful for: 2024-06-11T13:40:31.227 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch daemon add osd smithi135:vg_nvme/lv_1 2024-06-11T13:40:31.545 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:31 smithi135 ceph-mon[27511]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:31.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:31 smithi135 ceph-mon[27511]: osdmap e50: 7 total, 7 up, 7 in 2024-06-11T13:40:31.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:31 smithi135 ceph-mon[27511]: osdmap e51: 7 total, 7 up, 7 in 2024-06-11T13:40:31.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:31 smithi069 ceph-mon[25054]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:31.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:31 smithi069 ceph-mon[25054]: osdmap e50: 7 total, 7 up, 7 in 2024-06-11T13:40:31.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:31 smithi069 ceph-mon[25054]: osdmap e51: 7 total, 7 up, 7 in 2024-06-11T13:40:32.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: osdmap e52: 7 total, 7 up, 7 in 2024-06-11T13:40:32.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi135 2024-06-11T13:40:32.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi135 to 5312M 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:32 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: osdmap e52: 7 total, 7 up, 7 in 2024-06-11T13:40:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi135 2024-06-11T13:40:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:32.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:32.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi135 to 5312M 2024-06-11T13:40:32.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:32.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:32.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:32.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:32.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:32 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:33.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:33 smithi135 ceph-mon[27511]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:33.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:40:33.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:40:33.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:33 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:33.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:33 smithi069 ceph-mon[25054]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:33.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-06-11T13:40:33.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-06-11T13:40:33.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:33 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:34.708 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:34 smithi135 ceph-mon[27511]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:40:34.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:34 smithi069 ceph-mon[25054]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi135:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:40:35.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:35 smithi069 ceph-mon[25054]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:35 smithi135 ceph-mon[27511]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:36.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:36 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/1810856275' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "10174732-5da4-4ca0-ab3c-eacd34c2b3fb"}]: dispatch 2024-06-11T13:40:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:36 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "10174732-5da4-4ca0-ab3c-eacd34c2b3fb"}]: dispatch 2024-06-11T13:40:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:36 smithi069 ceph-mon[25054]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "10174732-5da4-4ca0-ab3c-eacd34c2b3fb"}]': finished 2024-06-11T13:40:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:36 smithi069 ceph-mon[25054]: osdmap e53: 8 total, 7 up, 8 in 2024-06-11T13:40:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:36 smithi069 ceph-mon[25054]: from='client.? 172.21.15.135:0/2411095808' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:40:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:36 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/1810856275' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "10174732-5da4-4ca0-ab3c-eacd34c2b3fb"}]: dispatch 2024-06-11T13:40:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:36 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "10174732-5da4-4ca0-ab3c-eacd34c2b3fb"}]: dispatch 2024-06-11T13:40:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:36 smithi135 ceph-mon[27511]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "10174732-5da4-4ca0-ab3c-eacd34c2b3fb"}]': finished 2024-06-11T13:40:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:36 smithi135 ceph-mon[27511]: osdmap e53: 8 total, 7 up, 8 in 2024-06-11T13:40:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:36 smithi135 ceph-mon[27511]: from='client.? 172.21.15.135:0/2411095808' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-06-11T13:40:37.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:37 smithi069 ceph-mon[25054]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-06-11T13:40:38.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:37 smithi135 ceph-mon[27511]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-06-11T13:40:39.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:39 smithi069 ceph-mon[25054]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-06-11T13:40:40.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:39 smithi135 ceph-mon[27511]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-06-11T13:40:41.862 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:41 smithi135 ceph-mon[27511]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-06-11T13:40:41.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:41 smithi069 ceph-mon[25054]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-06-11T13:40:42.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-11T13:40:42.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:42.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:42 smithi069 ceph-mon[25054]: Deploying daemon osd.7 on smithi135 2024-06-11T13:40:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-11T13:40:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:42 smithi135 ceph-mon[27511]: Deploying daemon osd.7 on smithi135 2024-06-11T13:40:43.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:43 smithi069 ceph-mon[25054]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:40:44.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:43 smithi135 ceph-mon[27511]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:40:45.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:45 smithi069 ceph-mon[25054]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:40:46.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:45 smithi135 ceph-mon[27511]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:40:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:47 smithi069 ceph-mon[25054]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:48.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:47 smithi135 ceph-mon[27511]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:48.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:48.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:48.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:48.232 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:50.023 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:49 smithi135 ceph-mon[27511]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:50.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:49 smithi069 ceph-mon[25054]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:50.448 INFO:teuthology.orchestra.run.smithi135.stdout:Created osd(s) 7 on host 'smithi135' 2024-06-11T13:40:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:40:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:40:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:50 smithi135 ceph-mon[27511]: from='osd.7 [v2:172.21.15.135:6824/302104920,v1:172.21.15.135:6825/302104920]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T13:40:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:50 smithi135 ceph-mon[27511]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T13:40:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:51.134 DEBUG:teuthology.orchestra.run.smithi135:osd.7> sudo journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.7.service 2024-06-11T13:40:51.137 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-06-11T13:40:51.137 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd stat -f json 2024-06-11T13:40:51.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:40:51.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:40:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:50 smithi069 ceph-mon[25054]: from='osd.7 [v2:172.21.15.135:6824/302104920,v1:172.21.15.135:6825/302104920]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T13:40:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:50 smithi069 ceph-mon[25054]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T13:40:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: osdmap e54: 8 total, 7 up, 8 in 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: from='osd.7 [v2:172.21.15.135:6824/302104920,v1:172.21.15.135:6825/302104920]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:51.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:52.267 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:52.268 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-11T13:40:52.268 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: osdmap e54: 8 total, 7 up, 8 in 2024-06-11T13:40:52.268 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:52.268 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: from='osd.7 [v2:172.21.15.135:6824/302104920,v1:172.21.15.135:6825/302104920]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:52.268 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T13:40:52.268 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:52.268 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[40710]: 2024-06-11T13:40:51.950+0000 7fadd6cc4700 -1 osd.7 0 waiting for initial osdmap 2024-06-11T13:40:52.268 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 13:40:51 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[40710]: 2024-06-11T13:40:51.958+0000 7fadd1e5c700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:40:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:52 smithi069 ceph-mon[25054]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:40:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:52 smithi069 ceph-mon[25054]: osdmap e55: 8 total, 7 up, 8 in 2024-06-11T13:40:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:53.101 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:52 smithi135 ceph-mon[27511]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]': finished 2024-06-11T13:40:53.101 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:52 smithi135 ceph-mon[27511]: osdmap e55: 8 total, 7 up, 8 in 2024-06-11T13:40:53.101 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:53.101 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:53.141 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:40:53.737 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1718113252,"num_in_osds":8,"osd_in_since":1718113235,"num_remapped_pgs":1} 2024-06-11T13:40:53.738 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-06-11T13:40:53.956 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:40:53.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: purged_snaps scrub starts 2024-06-11T13:40:53.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: purged_snaps scrub ok 2024-06-11T13:40:53.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:53.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: osd.7 [v2:172.21.15.135:6824/302104920,v1:172.21.15.135:6825/302104920] boot 2024-06-11T13:40:53.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: osdmap e56: 8 total, 8 up, 8 in 2024-06-11T13:40:53.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:53.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3822339301' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-11T13:40:53.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:53 smithi069 ceph-mon[25054]: osdmap e57: 8 total, 8 up, 8 in 2024-06-11T13:40:54.272 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: purged_snaps scrub starts 2024-06-11T13:40:54.272 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: purged_snaps scrub ok 2024-06-11T13:40:54.272 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-06-11T13:40:54.272 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: osd.7 [v2:172.21.15.135:6824/302104920,v1:172.21.15.135:6825/302104920] boot 2024-06-11T13:40:54.273 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: osdmap e56: 8 total, 8 up, 8 in 2024-06-11T13:40:54.273 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:40:54.273 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3822339301' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-06-11T13:40:54.273 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:53 smithi135 ceph-mon[27511]: osdmap e57: 8 total, 8 up, 8 in 2024-06-11T13:40:55.686 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:40:55.686 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":59,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","created":"2024-06-11T13:34:56.183002+0000","modified":"2024-06-11T13:40:55.530294+0000","last_up_change":"2024-06-11T13:40:52.944018+0000","last_in_change":"2024-06-11T13:40:35.727244+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-11T13:38:54.264314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4314153a-0a5d-44cb-b4fb-9e7e1c02118d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6803","nonce":33340411}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6805","nonce":33340411}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6809","nonce":33340411}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6807","nonce":33340411}]},"public_addr":"172.21.15.69:6803/33340411","cluster_addr":"172.21.15.69:6805/33340411","heartbeat_back_addr":"172.21.15.69:6809/33340411","heartbeat_front_addr":"172.21.15.69:6807/33340411","state":["exists","up"]},{"osd":1,"uuid":"d24d902f-6d71-4b9c-83d9-88cd91adae00","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6811","nonce":2008173180}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6813","nonce":2008173180}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6817","nonce":2008173180}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6815","nonce":2008173180}]},"public_addr":"172.21.15.69:6811/2008173180","cluster_addr":"172.21.15.69:6813/2008173180","heartbeat_back_addr":"172.21.15.69:6817/2008173180","heartbeat_front_addr":"172.21.15.69:6815/2008173180","state":["exists","up"]},{"osd":2,"uuid":"496987fe-a059-4050-9f86-df5ba85e744c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6819","nonce":2435076448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6821","nonce":2435076448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6825","nonce":2435076448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6823","nonce":2435076448}]},"public_addr":"172.21.15.69:6819/2435076448","cluster_addr":"172.21.15.69:6821/2435076448","heartbeat_back_addr":"172.21.15.69:6825/2435076448","heartbeat_front_addr":"172.21.15.69:6823/2435076448","state":["exists","up"]},{"osd":3,"uuid":"10b22a7a-0b19-4284-b8e1-276e1e80441f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6827","nonce":1053767534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6829","nonce":1053767534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6833","nonce":1053767534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6831","nonce":1053767534}]},"public_addr":"172.21.15.69:6827/1053767534","cluster_addr":"172.21.15.69:6829/1053767534","heartbeat_back_addr":"172.21.15.69:6833/1053767534","heartbeat_front_addr":"172.21.15.69:6831/1053767534","state":["exists","up"]},{"osd":4,"uuid":"c2eb3ce0-dac9-4447-97e9-ae8c1068503b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6801","nonce":90669583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6803","nonce":90669583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6807","nonce":90669583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6805","nonce":90669583}]},"public_addr":"172.21.15.135:6801/90669583","cluster_addr":"172.21.15.135:6803/90669583","heartbeat_back_addr":"172.21.15.135:6807/90669583","heartbeat_front_addr":"172.21.15.135:6805/90669583","state":["exists","up"]},{"osd":5,"uuid":"7a8e63d3-9163-464c-86a6-4ceb5ed183a5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6809","nonce":3461261274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6811","nonce":3461261274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6815","nonce":3461261274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6813","nonce":3461261274}]},"public_addr":"172.21.15.135:6809/3461261274","cluster_addr":"172.21.15.135:6811/3461261274","heartbeat_back_addr":"172.21.15.135:6815/3461261274","heartbeat_front_addr":"172.21.15.135:6813/3461261274","state":["exists","up"]},{"osd":6,"uuid":"2743be9d-a84a-431f-8afe-6c99cd0b6b0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6817","nonce":1667247705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6819","nonce":1667247705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6823","nonce":1667247705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6821","nonce":1667247705}]},"public_addr":"172.21.15.135:6817/1667247705","cluster_addr":"172.21.15.135:6819/1667247705","heartbeat_back_addr":"172.21.15.135:6823/1667247705","heartbeat_front_addr":"172.21.15.135:6821/1667247705","state":["exists","up"]},{"osd":7,"uuid":"10174732-5da4-4ca0-ab3c-eacd34c2b3fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6825","nonce":302104920}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6827","nonce":302104920}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6831","nonce":302104920}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6829","nonce":302104920}]},"public_addr":"172.21.15.135:6825/302104920","cluster_addr":"172.21.15.135:6827/302104920","heartbeat_back_addr":"172.21.15.135:6831/302104920","heartbeat_front_addr":"172.21.15.135:6829/302104920","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:37:52.868679+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:38:21.769523+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:38:50.919417+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:39:17.051867+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:39:38.351921+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:01.967825+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:26.448992+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:51.021115+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.69:0/382903734":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6800/4193674661":"2024-06-12T13:35:57.540925+0000","172.21.15.69:0/2134476654":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6800/167955139":"2024-06-12T13:36:49.906867+0000","172.21.15.69:0/3735048043":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6801/167955139":"2024-06-12T13:36:49.906867+0000","172.21.15.69:0/2847944718":"2024-06-12T13:35:57.540925+0000","172.21.15.69:0/2772116633":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6800/2509713441":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/2771700286":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6801/4193674661":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6801/2509713441":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/1624426066":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/276566878":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/554963710":"2024-06-12T13:35:28.478847+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-11T13:40:55.686 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: Detected new or changed devices on smithi135 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi135 to 3984M 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:55.687 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:55.688 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:55.688 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:55.688 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: osdmap e58: 8 total, 8 up, 8 in 2024-06-11T13:40:55.688 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:55 smithi069 ceph-mon[25054]: osdmap e59: 8 total, 8 up, 8 in 2024-06-11T13:40:56.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: Detected new or changed devices on smithi135 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi135 to 3984M 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:40:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:40:56.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:40:56.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: osdmap e58: 8 total, 8 up, 8 in 2024-06-11T13:40:56.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:55 smithi135 ceph-mon[27511]: osdmap e59: 8 total, 8 up, 8 in 2024-06-11T13:40:56.305 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-06-11T13:38:54.264314+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-06-11T13:40:56.305 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-06-11T13:40:56.521 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:40:56.873 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:56 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/872178430' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-11T13:40:56.873 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:56 smithi069 ceph-mon[25054]: osdmap e60: 8 total, 8 up, 8 in 2024-06-11T13:40:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:56 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/872178430' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-11T13:40:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:56 smithi135 ceph-mon[27511]: osdmap e60: 8 total, 8 up, 8 in 2024-06-11T13:40:57.901 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:57 smithi069 ceph-mon[25054]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:40:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:57 smithi135 ceph-mon[27511]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:40:58.287 INFO:teuthology.orchestra.run.smithi069.stdout:pg_num: 1 2024-06-11T13:40:58.875 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:58 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1551786411' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-11T13:40:58.902 INFO:tasks.cephadm:Setting up client nodes... 2024-06-11T13:40:58.903 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-06-11T13:40:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:58 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1551786411' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-06-11T13:40:59.907 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:40:59 smithi069 ceph-mon[25054]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-06-11T13:41:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:40:59 smithi135 ceph-mon[27511]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-06-11T13:41:01.017 INFO:teuthology.orchestra.run.smithi069.stdout:[client.0] 2024-06-11T13:41:01.017 INFO:teuthology.orchestra.run.smithi069.stdout: key = AQDtU2hmwlf5ABAAEAd4zTF4ZsRbflHArTWzAw== 2024-06-11T13:41:01.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:01 smithi069 ceph-mon[25054]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-11T13:41:01.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:01 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3823888560' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-11T13:41:01.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:01 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3823888560' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-11T13:41:01.779 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-06-11T13:41:01.779 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-06-11T13:41:01.779 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-06-11T13:41:01.820 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-06-11T13:41:01.820 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-06-11T13:41:01.821 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph mgr dump --format=json 2024-06-11T13:41:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:01 smithi135 ceph-mon[27511]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-06-11T13:41:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:01 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3823888560' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-06-11T13:41:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:01 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3823888560' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-06-11T13:41:02.075 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:03.804 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:03 smithi069 ceph-mon[25054]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-11T13:41:03.987 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:41:04.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:03 smithi135 ceph-mon[27511]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-06-11T13:41:04.759 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi069.iruzyw","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6800","nonce":3418329770},{"type":"v1","addr":"172.21.15.69:6801","nonce":3418329770}]},"active_addr":"172.21.15.69:6801/3418329770","active_change":"2024-06-11T13:36:49.907152+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14226,"name":"smithi135.xcould","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.69:8443/","prometheus":"http://172.21.15.69:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":92441976}]},{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":2970851797}]},{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":2377342647}]},{"addrvec":[{"type":"v2","addr":"172.21.15.69:0","nonce":2582524983}]}]}} 2024-06-11T13:41:04.768 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-06-11T13:41:04.768 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-06-11T13:41:04.769 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-06-11T13:41:04.981 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:04.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:04 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3800500688' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-11T13:41:05.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:04 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3800500688' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-06-11T13:41:05.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:05 smithi069 ceph-mon[25054]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-06-11T13:41:06.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:05 smithi135 ceph-mon[27511]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-06-11T13:41:06.689 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:41:06.689 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":60,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","created":"2024-06-11T13:34:56.183002+0000","modified":"2024-06-11T13:40:56.534717+0000","last_up_change":"2024-06-11T13:40:52.944018+0000","last_in_change":"2024-06-11T13:40:35.727244+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-11T13:38:54.264314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4314153a-0a5d-44cb-b4fb-9e7e1c02118d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6803","nonce":33340411}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6805","nonce":33340411}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6809","nonce":33340411}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6807","nonce":33340411}]},"public_addr":"172.21.15.69:6803/33340411","cluster_addr":"172.21.15.69:6805/33340411","heartbeat_back_addr":"172.21.15.69:6809/33340411","heartbeat_front_addr":"172.21.15.69:6807/33340411","state":["exists","up"]},{"osd":1,"uuid":"d24d902f-6d71-4b9c-83d9-88cd91adae00","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6811","nonce":2008173180}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6813","nonce":2008173180}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6817","nonce":2008173180}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6815","nonce":2008173180}]},"public_addr":"172.21.15.69:6811/2008173180","cluster_addr":"172.21.15.69:6813/2008173180","heartbeat_back_addr":"172.21.15.69:6817/2008173180","heartbeat_front_addr":"172.21.15.69:6815/2008173180","state":["exists","up"]},{"osd":2,"uuid":"496987fe-a059-4050-9f86-df5ba85e744c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6819","nonce":2435076448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6821","nonce":2435076448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6825","nonce":2435076448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6823","nonce":2435076448}]},"public_addr":"172.21.15.69:6819/2435076448","cluster_addr":"172.21.15.69:6821/2435076448","heartbeat_back_addr":"172.21.15.69:6825/2435076448","heartbeat_front_addr":"172.21.15.69:6823/2435076448","state":["exists","up"]},{"osd":3,"uuid":"10b22a7a-0b19-4284-b8e1-276e1e80441f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6827","nonce":1053767534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6829","nonce":1053767534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6833","nonce":1053767534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6831","nonce":1053767534}]},"public_addr":"172.21.15.69:6827/1053767534","cluster_addr":"172.21.15.69:6829/1053767534","heartbeat_back_addr":"172.21.15.69:6833/1053767534","heartbeat_front_addr":"172.21.15.69:6831/1053767534","state":["exists","up"]},{"osd":4,"uuid":"c2eb3ce0-dac9-4447-97e9-ae8c1068503b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6801","nonce":90669583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6803","nonce":90669583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6807","nonce":90669583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6805","nonce":90669583}]},"public_addr":"172.21.15.135:6801/90669583","cluster_addr":"172.21.15.135:6803/90669583","heartbeat_back_addr":"172.21.15.135:6807/90669583","heartbeat_front_addr":"172.21.15.135:6805/90669583","state":["exists","up"]},{"osd":5,"uuid":"7a8e63d3-9163-464c-86a6-4ceb5ed183a5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6809","nonce":3461261274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6811","nonce":3461261274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6815","nonce":3461261274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6813","nonce":3461261274}]},"public_addr":"172.21.15.135:6809/3461261274","cluster_addr":"172.21.15.135:6811/3461261274","heartbeat_back_addr":"172.21.15.135:6815/3461261274","heartbeat_front_addr":"172.21.15.135:6813/3461261274","state":["exists","up"]},{"osd":6,"uuid":"2743be9d-a84a-431f-8afe-6c99cd0b6b0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6817","nonce":1667247705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6819","nonce":1667247705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6823","nonce":1667247705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6821","nonce":1667247705}]},"public_addr":"172.21.15.135:6817/1667247705","cluster_addr":"172.21.15.135:6819/1667247705","heartbeat_back_addr":"172.21.15.135:6823/1667247705","heartbeat_front_addr":"172.21.15.135:6821/1667247705","state":["exists","up"]},{"osd":7,"uuid":"10174732-5da4-4ca0-ab3c-eacd34c2b3fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6825","nonce":302104920}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6827","nonce":302104920}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6831","nonce":302104920}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6829","nonce":302104920}]},"public_addr":"172.21.15.135:6825/302104920","cluster_addr":"172.21.15.135:6827/302104920","heartbeat_back_addr":"172.21.15.135:6831/302104920","heartbeat_front_addr":"172.21.15.135:6829/302104920","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:37:52.868679+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:38:21.769523+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:38:50.919417+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:39:17.051867+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:39:38.351921+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:01.967825+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:26.448992+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:51.021115+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.69:0/382903734":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6800/4193674661":"2024-06-12T13:35:57.540925+0000","172.21.15.69:0/2134476654":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6800/167955139":"2024-06-12T13:36:49.906867+0000","172.21.15.69:0/3735048043":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6801/167955139":"2024-06-12T13:36:49.906867+0000","172.21.15.69:0/2847944718":"2024-06-12T13:35:57.540925+0000","172.21.15.69:0/2772116633":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6800/2509713441":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/2771700286":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6801/4193674661":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6801/2509713441":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/1624426066":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/276566878":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/554963710":"2024-06-12T13:35:28.478847+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-11T13:41:07.370 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-06-11T13:41:07.370 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd dump --format=json 2024-06-11T13:41:07.586 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:07.708 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:07 smithi069 ceph-mon[25054]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:41:07.708 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:07 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3012928797' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-11T13:41:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:07 smithi135 ceph-mon[27511]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-06-11T13:41:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:07 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3012928797' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-11T13:41:09.316 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:41:09.316 INFO:teuthology.orchestra.run.smithi069.stdout:{"epoch":60,"fsid":"5368e33c-27f7-11ef-bc9b-c7b262605968","created":"2024-06-11T13:34:56.183002+0000","modified":"2024-06-11T13:40:56.534717+0000","last_up_change":"2024-06-11T13:40:52.944018+0000","last_in_change":"2024-06-11T13:40:35.727244+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-06-11T13:38:54.264314+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"4314153a-0a5d-44cb-b4fb-9e7e1c02118d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6802","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6803","nonce":33340411}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6804","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6805","nonce":33340411}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6808","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6809","nonce":33340411}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6806","nonce":33340411},{"type":"v1","addr":"172.21.15.69:6807","nonce":33340411}]},"public_addr":"172.21.15.69:6803/33340411","cluster_addr":"172.21.15.69:6805/33340411","heartbeat_back_addr":"172.21.15.69:6809/33340411","heartbeat_front_addr":"172.21.15.69:6807/33340411","state":["exists","up"]},{"osd":1,"uuid":"d24d902f-6d71-4b9c-83d9-88cd91adae00","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6810","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6811","nonce":2008173180}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6812","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6813","nonce":2008173180}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6816","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6817","nonce":2008173180}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6814","nonce":2008173180},{"type":"v1","addr":"172.21.15.69:6815","nonce":2008173180}]},"public_addr":"172.21.15.69:6811/2008173180","cluster_addr":"172.21.15.69:6813/2008173180","heartbeat_back_addr":"172.21.15.69:6817/2008173180","heartbeat_front_addr":"172.21.15.69:6815/2008173180","state":["exists","up"]},{"osd":2,"uuid":"496987fe-a059-4050-9f86-df5ba85e744c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6818","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6819","nonce":2435076448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6820","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6821","nonce":2435076448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6824","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6825","nonce":2435076448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6822","nonce":2435076448},{"type":"v1","addr":"172.21.15.69:6823","nonce":2435076448}]},"public_addr":"172.21.15.69:6819/2435076448","cluster_addr":"172.21.15.69:6821/2435076448","heartbeat_back_addr":"172.21.15.69:6825/2435076448","heartbeat_front_addr":"172.21.15.69:6823/2435076448","state":["exists","up"]},{"osd":3,"uuid":"10b22a7a-0b19-4284-b8e1-276e1e80441f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6826","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6827","nonce":1053767534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6828","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6829","nonce":1053767534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6832","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6833","nonce":1053767534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:6830","nonce":1053767534},{"type":"v1","addr":"172.21.15.69:6831","nonce":1053767534}]},"public_addr":"172.21.15.69:6827/1053767534","cluster_addr":"172.21.15.69:6829/1053767534","heartbeat_back_addr":"172.21.15.69:6833/1053767534","heartbeat_front_addr":"172.21.15.69:6831/1053767534","state":["exists","up"]},{"osd":4,"uuid":"c2eb3ce0-dac9-4447-97e9-ae8c1068503b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6800","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6801","nonce":90669583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6802","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6803","nonce":90669583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6806","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6807","nonce":90669583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6804","nonce":90669583},{"type":"v1","addr":"172.21.15.135:6805","nonce":90669583}]},"public_addr":"172.21.15.135:6801/90669583","cluster_addr":"172.21.15.135:6803/90669583","heartbeat_back_addr":"172.21.15.135:6807/90669583","heartbeat_front_addr":"172.21.15.135:6805/90669583","state":["exists","up"]},{"osd":5,"uuid":"7a8e63d3-9163-464c-86a6-4ceb5ed183a5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6808","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6809","nonce":3461261274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6810","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6811","nonce":3461261274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6814","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6815","nonce":3461261274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6812","nonce":3461261274},{"type":"v1","addr":"172.21.15.135:6813","nonce":3461261274}]},"public_addr":"172.21.15.135:6809/3461261274","cluster_addr":"172.21.15.135:6811/3461261274","heartbeat_back_addr":"172.21.15.135:6815/3461261274","heartbeat_front_addr":"172.21.15.135:6813/3461261274","state":["exists","up"]},{"osd":6,"uuid":"2743be9d-a84a-431f-8afe-6c99cd0b6b0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6816","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6817","nonce":1667247705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6818","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6819","nonce":1667247705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6822","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6823","nonce":1667247705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6820","nonce":1667247705},{"type":"v1","addr":"172.21.15.135:6821","nonce":1667247705}]},"public_addr":"172.21.15.135:6817/1667247705","cluster_addr":"172.21.15.135:6819/1667247705","heartbeat_back_addr":"172.21.15.135:6823/1667247705","heartbeat_front_addr":"172.21.15.135:6821/1667247705","state":["exists","up"]},{"osd":7,"uuid":"10174732-5da4-4ca0-ab3c-eacd34c2b3fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6824","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6825","nonce":302104920}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6826","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6827","nonce":302104920}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6830","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6831","nonce":302104920}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.135:6828","nonce":302104920},{"type":"v1","addr":"172.21.15.135:6829","nonce":302104920}]},"public_addr":"172.21.15.135:6825/302104920","cluster_addr":"172.21.15.135:6827/302104920","heartbeat_back_addr":"172.21.15.135:6831/302104920","heartbeat_front_addr":"172.21.15.135:6829/302104920","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:37:52.868679+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:38:21.769523+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:38:50.919417+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:39:17.051867+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:39:38.351921+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:01.967825+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:26.448992+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-06-11T13:40:51.021115+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.69:0/382903734":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6800/4193674661":"2024-06-12T13:35:57.540925+0000","172.21.15.69:0/2134476654":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6800/167955139":"2024-06-12T13:36:49.906867+0000","172.21.15.69:0/3735048043":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6801/167955139":"2024-06-12T13:36:49.906867+0000","172.21.15.69:0/2847944718":"2024-06-12T13:35:57.540925+0000","172.21.15.69:0/2772116633":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6800/2509713441":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/2771700286":"2024-06-12T13:36:49.906867+0000","172.21.15.69:6801/4193674661":"2024-06-12T13:35:57.540925+0000","172.21.15.69:6801/2509713441":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/1624426066":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/276566878":"2024-06-12T13:35:28.478847+0000","172.21.15.69:0/554963710":"2024-06-12T13:35:28.478847+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-06-11T13:41:09.848 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:09 smithi135 ceph-mon[27511]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-06-11T13:41:09.848 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:09 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2214095607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-11T13:41:09.989 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:09 smithi069 ceph-mon[25054]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-06-11T13:41:09.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:09 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2214095607' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-06-11T13:41:10.017 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-06-11T13:41:10.018 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-06-11T13:41:10.018 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-06-11T13:41:10.018 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-06-11T13:41:10.019 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-06-11T13:41:10.019 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-06-11T13:41:10.020 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-06-11T13:41:10.020 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-06-11T13:41:10.435 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.439 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.440 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.441 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.444 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.445 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.451 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:10.453 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:11.809 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:11 smithi069 ceph-mon[25054]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:11 smithi135 ceph-mon[27511]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:13.848 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:13 smithi069 ceph-mon[25054]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:13 smithi135 ceph-mon[27511]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:15.810 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:15 smithi069 ceph-mon[25054]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:15.836 INFO:teuthology.orchestra.run.smithi069.stdout:60129542181 2024-06-11T13:41:15.837 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-06-11T13:41:16.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:15 smithi135 ceph-mon[27511]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:17.714 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:17 smithi069 ceph-mon[25054]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:18.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:17 smithi135 ceph-mon[27511]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:19.835 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:19 smithi069 ceph-mon[25054]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:19 smithi135 ceph-mon[27511]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:20.832 INFO:teuthology.orchestra.run.smithi069.stdout:38654705708 2024-06-11T13:41:20.832 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-06-11T13:41:21.932 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:21 smithi069 ceph-mon[25054]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:21 smithi135 ceph-mon[27511]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:22.516 INFO:teuthology.orchestra.run.smithi069.stdout:81604378656 2024-06-11T13:41:22.516 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-06-11T13:41:23.109 INFO:teuthology.orchestra.run.smithi069.stdout:171798691858 2024-06-11T13:41:23.110 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-06-11T13:41:23.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:23 smithi069 ceph-mon[25054]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:23.847 INFO:teuthology.orchestra.run.smithi069.stdout:115964117019 2024-06-11T13:41:23.847 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-06-11T13:41:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:23 smithi135 ceph-mon[27511]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:24.199 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:24.199 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:24.680 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:25.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:25 smithi069 ceph-mon[25054]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:25 smithi135 ceph-mon[27511]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:27.744 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:27 smithi069 ceph-mon[25054]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:27.821 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:27 smithi135 ceph-mon[27511]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:28.395 INFO:teuthology.orchestra.run.smithi069.stdout:206158430223 2024-06-11T13:41:28.396 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-06-11T13:41:28.797 INFO:teuthology.orchestra.run.smithi069.stdout:240518168585 2024-06-11T13:41:28.798 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-06-11T13:41:29.874 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:29 smithi069 ceph-mon[25054]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:29 smithi135 ceph-mon[27511]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:30.818 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:31.028 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:31.028 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:31.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:31 smithi069 ceph-mon[25054]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:31 smithi135 ceph-mon[27511]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:32.819 INFO:teuthology.orchestra.run.smithi069.stdout:171798691860 2024-06-11T13:41:33.579 INFO:teuthology.orchestra.run.smithi069.stdout:60129542185 2024-06-11T13:41:33.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:33 smithi069 ceph-mon[25054]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:33.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:33 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3744737890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-11T13:41:33.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:33 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2126190452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-11T13:41:34.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:33 smithi135 ceph-mon[27511]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:34.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:33 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3744737890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-06-11T13:41:34.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:33 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2126190452' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-06-11T13:41:34.916 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542181 got 60129542185 for osd.1 2024-06-11T13:41:34.917 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:35.184 INFO:teuthology.orchestra.run.smithi069.stdout:137438953497 2024-06-11T13:41:35.184 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-06-11T13:41:35.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:35 smithi069 ceph-mon[25054]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:35 smithi135 ceph-mon[27511]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:36.683 INFO:teuthology.orchestra.run.smithi069.stdout:81604378660 2024-06-11T13:41:37.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:37 smithi069 ceph-mon[25054]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:37.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:37 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/814068249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-11T13:41:37.925 INFO:teuthology.orchestra.run.smithi069.stdout:240518168587 2024-06-11T13:41:38.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:37 smithi135 ceph-mon[27511]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:38.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:37 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/814068249' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-06-11T13:41:38.137 INFO:teuthology.orchestra.run.smithi069.stdout:206158430225 2024-06-11T13:41:38.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:38 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1787425328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-11T13:41:38.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:38 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/631447018' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-11T13:41:39.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:38 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1787425328' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-06-11T13:41:39.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:38 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/631447018' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-06-11T13:41:39.443 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:39.467 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168587 for osd.7 2024-06-11T13:41:39.467 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:39.689 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:39 smithi069 ceph-mon[25054]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:39.948 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:39 smithi135 ceph-mon[27511]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:40.072 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691858 got 171798691860 for osd.5 2024-06-11T13:41:40.072 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:40.443 INFO:teuthology.orchestra.run.smithi069.stdout:38654705712 2024-06-11T13:41:40.885 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:40 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1506365124' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-11T13:41:40.887 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378656 got 81604378660 for osd.2 2024-06-11T13:41:40.887 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:41.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:40 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1506365124' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-06-11T13:41:41.451 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705712 for osd.0 2024-06-11T13:41:41.451 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:41.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:41 smithi069 ceph-mon[25054]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:41.812 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430223 got 206158430225 for osd.6 2024-06-11T13:41:41.812 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:41 smithi135 ceph-mon[27511]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:42.900 INFO:teuthology.orchestra.run.smithi069.stdout:115964117023 2024-06-11T13:41:43.692 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117019 got 115964117023 for osd.3 2024-06-11T13:41:43.692 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:43.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:43 smithi069 ceph-mon[25054]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:43.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:43 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1905558410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-11T13:41:44.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:43 smithi135 ceph-mon[27511]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:44.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:43 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1905558410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-06-11T13:41:44.298 INFO:teuthology.orchestra.run.smithi069.stdout:137438953500 2024-06-11T13:41:44.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:44 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/115258153' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-11T13:41:44.882 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953497 got 137438953500 for osd.4 2024-06-11T13:41:44.883 DEBUG:teuthology.parallel:result is None 2024-06-11T13:41:44.883 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-06-11T13:41:44.883 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph pg dump --format=json 2024-06-11T13:41:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:44 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/115258153' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-06-11T13:41:45.096 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:45.857 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:45 smithi069 ceph-mon[25054]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:46.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:45 smithi135 ceph-mon[27511]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:46.754 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:41:46.756 INFO:teuthology.orchestra.run.smithi069.stderr:dumped all 2024-06-11T13:41:47.472 INFO:teuthology.orchestra.run.smithi069.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2024-06-11T13:41:46.011031+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48936,"kb_used_data":6376,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912280,"statfs":{"total":767960285184,"available":767910174720,"internally_reserved":0,"allocated":6529024,"data_stored":3571802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002192"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-11T13:40:56.542367+0000","last_change":"2024-06-11T13:40:56.542367+0000","last_active":"2024-06-11T13:40:56.542367+0000","last_peered":"2024-06-11T13:40:56.542367+0000","last_clean":"2024-06-11T13:40:56.542367+0000","last_became_active":"2024-06-11T13:40:56.541922+0000","last_became_peered":"2024-06-11T13:40:56.541922+0000","last_unstale":"2024-06-11T13:40:56.542367+0000","last_undegraded":"2024-06-11T13:40:56.542367+0000","last_fullsized":"2024-06-11T13:40:56.542367+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-11T13:38:54.879429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-11T13:38:54.879429+0000","last_clean_scrub_stamp":"2024-06-11T13:38:54.879429+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-13T00:27:38.527587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66800000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46800000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69099999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70099999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65900000000000003}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.59899999999999998}]},{"osd":1,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.67200000000000004}]},{"osd":2,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.624},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.63200000000000001}]},{"osd":3,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.64800000000000002}]},{"osd":4,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.439}]},{"osd":5,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.49199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52100000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.32100000000000001},"min":{"1min":0.22900000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.56200000000000006,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.438},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.34699999999999998,"15min":0.32100000000000001},"min":{"1min":0.247,"5min":0.224,"15min":0.224},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.45400000000000001}]},{"osd":2,"last update":"Tue Jun 11 13:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39900000000000002,"15min":0.39500000000000002},"min":{"1min":0.32700000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.40699999999999997}]},{"osd":3,"last update":"Tue Jun 11 13:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.45100000000000001,"15min":0.45300000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.57999999999999996,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.47999999999999998}]},{"osd":4,"last update":"Tue Jun 11 13:40:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.83599999999999997}]},{"osd":5,"last update":"Tue Jun 11 13:41:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.78400000000000003}]},{"osd":6,"last update":"Tue Jun 11 13:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.88400000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Jun 11 13:41:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.29699999999999999},"min":{"1min":0.252,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.54200000000000004,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.34100000000000003,"15min":0.30599999999999999},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.24299999999999999}]},{"osd":2,"last update":"Tue Jun 11 13:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38100000000000001,"15min":0.373},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.38800000000000001,"15min":0.379},"min":{"1min":0.29899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.316}]},{"osd":3,"last update":"Tue Jun 11 13:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.433,"15min":0.439},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.51800000000000002,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.47699999999999998,"15min":0.48299999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.60799999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Tue Jun 11 13:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.66000000000000003,"5min":0.871,"15min":0.871},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.7749999999999999,"5min":0.84499999999999997,"15min":0.68999999999999995},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":19.190000000000001,"5min":19.190000000000001,"15min":19.190000000000001},"last":0.60799999999999998}]},{"osd":5,"last update":"Tue Jun 11 13:41:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.623}]},{"osd":6,"last update":"Tue Jun 11 13:41:29 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.71299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.27900000000000003,"5min":0.219,"15min":0.219},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.317},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.35999999999999999},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.41799999999999998}]},{"osd":1,"last update":"Tue Jun 11 13:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.371},{"interface":"front","average":{"1min":0.44,"5min":0.375,"15min":0.36399999999999999},"min":{"1min":0.311,"5min":0.253,"15min":0.253},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.28899999999999998}]},{"osd":3,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47199999999999998,"15min":0.47099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46400000000000002,"15min":0.46500000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.35499999999999998}]},{"osd":4,"last update":"Tue Jun 11 13:41:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.58499999999999996,"15min":0.58299999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.54700000000000004,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.64800000000000002}]},{"osd":5,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.78700000000000003}]},{"osd":6,"last update":"Tue Jun 11 13:41:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.63800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67200000000000004}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40000000000000002,"15min":0.40400000000000003},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41599999999999998,"15min":0.41699999999999998},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.30499999999999999}]},{"osd":1,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.41399999999999998,"15min":0.41599999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.55100000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.432,"15min":0.438},"min":{"1min":0.28799999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.50700000000000001,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.35599999999999998}]},{"osd":2,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.45600000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.55400000000000005,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.55900000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.27400000000000002}]},{"osd":4,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.44700000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79100000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66000000000000003}]},{"osd":5,"last update":"Tue Jun 11 13:41:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.627},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.58099999999999996}]},{"osd":6,"last update":"Tue Jun 11 13:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.68700000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.54300000000000004,"15min":0.52600000000000002},"min":{"1min":0.51100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54700000000000004,"15min":0.54200000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.79200000000000004}]},{"osd":1,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.53500000000000003,"15min":0.52600000000000002},"min":{"1min":0.42299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.53400000000000003,"15min":0.51800000000000002},"min":{"1min":0.40899999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.76800000000000002}]},{"osd":2,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.56100000000000005,"15min":0.55400000000000005},"min":{"1min":0.42399999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.755,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.52700000000000002,"15min":0.51700000000000002},"min":{"1min":0.435,"5min":0.374,"15min":0.374},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.64800000000000002}]},{"osd":3,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.52400000000000002,"15min":0.51200000000000001},"min":{"1min":0.432,"5min":0.374,"15min":0.374},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53600000000000003,"15min":0.52700000000000002},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.69199999999999995}]},{"osd":5,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.69999999999999996}]},{"osd":6,"last update":"Tue Jun 11 13:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.54300000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74199999999999999}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.61899999999999999}]},{"osd":1,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.63900000000000001}]},{"osd":2,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.53400000000000003}]},{"osd":3,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.47899999999999998}]},{"osd":4,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.60999999999999999}]},{"osd":6,"last update":"Tue Jun 11 13:41:29 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.51500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-11T13:41:47.473 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph pg dump --format=json 2024-06-11T13:41:47.685 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:47.744 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:47 smithi069 ceph-mon[25054]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:47.744 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:47 smithi069 ceph-mon[25054]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:41:48.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:47 smithi135 ceph-mon[27511]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:48.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:47 smithi135 ceph-mon[27511]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:41:49.350 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:41:49.351 INFO:teuthology.orchestra.run.smithi069.stderr:dumped all 2024-06-11T13:41:49.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:49 smithi069 ceph-mon[25054]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:49.980 INFO:teuthology.orchestra.run.smithi069.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-06-11T13:41:48.011430+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48936,"kb_used_data":6376,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912280,"statfs":{"total":767960285184,"available":767910174720,"internally_reserved":0,"allocated":6529024,"data_stored":3571802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002246"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-06-11T13:40:56.542367+0000","last_change":"2024-06-11T13:40:56.542367+0000","last_active":"2024-06-11T13:40:56.542367+0000","last_peered":"2024-06-11T13:40:56.542367+0000","last_clean":"2024-06-11T13:40:56.542367+0000","last_became_active":"2024-06-11T13:40:56.541922+0000","last_became_peered":"2024-06-11T13:40:56.541922+0000","last_unstale":"2024-06-11T13:40:56.542367+0000","last_undegraded":"2024-06-11T13:40:56.542367+0000","last_fullsized":"2024-06-11T13:40:56.542367+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-06-11T13:38:54.879429+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-06-11T13:38:54.879429+0000","last_clean_scrub_stamp":"2024-06-11T13:38:54.879429+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-06-13T00:27:38.527587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51600000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44400000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39600000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36499999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56399999999999995}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6184,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738968,"statfs":{"total":95995035648,"available":95988703232,"internally_reserved":0,"allocated":1146880,"data_stored":776930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.59899999999999998}]},{"osd":1,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.67200000000000004}]},{"osd":2,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.624},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.63200000000000001}]},{"osd":3,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.64800000000000002}]},{"osd":4,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.439}]},{"osd":5,"last update":"Tue Jun 11 13:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.49199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52100000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.34699999999999998,"15min":0.32100000000000001},"min":{"1min":0.22900000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.56200000000000006,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.438},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.34699999999999998,"15min":0.32100000000000001},"min":{"1min":0.247,"5min":0.224,"15min":0.224},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.45400000000000001}]},{"osd":2,"last update":"Tue Jun 11 13:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.39900000000000002,"15min":0.39500000000000002},"min":{"1min":0.32700000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.40699999999999997}]},{"osd":3,"last update":"Tue Jun 11 13:41:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.45100000000000001,"15min":0.45300000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.57999999999999996,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.47999999999999998}]},{"osd":4,"last update":"Tue Jun 11 13:40:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.83599999999999997}]},{"osd":5,"last update":"Tue Jun 11 13:41:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.78400000000000003}]},{"osd":6,"last update":"Tue Jun 11 13:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64200000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.88400000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Jun 11 13:41:29 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.33100000000000002,"15min":0.29699999999999999},"min":{"1min":0.252,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.54200000000000004,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.34100000000000003,"15min":0.30599999999999999},"min":{"1min":0.23499999999999999,"5min":0.219,"15min":0.219},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.24299999999999999}]},{"osd":2,"last update":"Tue Jun 11 13:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38100000000000001,"15min":0.373},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.38800000000000001,"15min":0.379},"min":{"1min":0.29899999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.56499999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.316}]},{"osd":3,"last update":"Tue Jun 11 13:41:25 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.433,"15min":0.439},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.51800000000000002,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.47699999999999998,"15min":0.48299999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.60799999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Tue Jun 11 13:41:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.66000000000000003,"5min":0.871,"15min":0.871},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.7749999999999999,"5min":0.84499999999999997,"15min":0.68999999999999995},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":19.190000000000001,"5min":19.190000000000001,"15min":19.190000000000001},"last":0.60799999999999998}]},{"osd":5,"last update":"Tue Jun 11 13:41:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.623}]},{"osd":6,"last update":"Tue Jun 11 13:41:29 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.71299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.27900000000000003,"5min":0.219,"15min":0.219},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.317},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.35999999999999999},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.41799999999999998}]},{"osd":1,"last update":"Tue Jun 11 13:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.28999999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.371},{"interface":"front","average":{"1min":0.44,"5min":0.375,"15min":0.36399999999999999},"min":{"1min":0.311,"5min":0.253,"15min":0.253},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.28899999999999998}]},{"osd":3,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.47199999999999998,"15min":0.47099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.0660000000000001,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46400000000000002,"15min":0.46500000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.35499999999999998}]},{"osd":4,"last update":"Tue Jun 11 13:41:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.58499999999999996,"15min":0.58299999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.54700000000000004,"15min":0.54000000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.64800000000000002}]},{"osd":5,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.78700000000000003}]},{"osd":6,"last update":"Tue Jun 11 13:41:28 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.63800000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67200000000000004}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40000000000000002,"15min":0.40400000000000003},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41599999999999998,"15min":0.41699999999999998},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.30499999999999999}]},{"osd":1,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.41399999999999998,"15min":0.41599999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.55100000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.432,"15min":0.438},"min":{"1min":0.28799999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.50700000000000001,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.35599999999999998}]},{"osd":2,"last update":"Tue Jun 11 13:41:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.45600000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.55400000000000005,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.436,"15min":0.44},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.55900000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.27400000000000002}]},{"osd":4,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.44700000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79100000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66000000000000003}]},{"osd":5,"last update":"Tue Jun 11 13:41:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.627},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.58099999999999996}]},{"osd":6,"last update":"Tue Jun 11 13:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.68700000000000006}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.54300000000000004,"15min":0.52600000000000002},"min":{"1min":0.51100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54700000000000004,"15min":0.54200000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.79200000000000004}]},{"osd":1,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.53500000000000003,"15min":0.52600000000000002},"min":{"1min":0.42299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.53400000000000003,"15min":0.51800000000000002},"min":{"1min":0.40899999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.76800000000000002}]},{"osd":2,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.56100000000000005,"15min":0.55400000000000005},"min":{"1min":0.42399999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.755,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.52700000000000002,"15min":0.51700000000000002},"min":{"1min":0.435,"5min":0.374,"15min":0.374},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.64800000000000002}]},{"osd":3,"last update":"Tue Jun 11 13:41:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.52400000000000002,"15min":0.51200000000000001},"min":{"1min":0.432,"5min":0.374,"15min":0.374},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53600000000000003,"15min":0.52700000000000002},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.69199999999999995}]},{"osd":5,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.69999999999999996}]},{"osd":6,"last update":"Tue Jun 11 13:41:30 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.54300000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.74199999999999999}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.61899999999999999}]},{"osd":1,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.63900000000000001}]},{"osd":2,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.53400000000000003}]},{"osd":3,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.47899999999999998}]},{"osd":4,"last update":"Tue Jun 11 13:41:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.60999999999999999}]},{"osd":6,"last update":"Tue Jun 11 13:41:29 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.51500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55700000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-06-11T13:41:49.982 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-06-11T13:41:49.982 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-06-11T13:41:49.982 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-06-11T13:41:49.982 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph health --format=json 2024-06-11T13:41:50.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:49 smithi135 ceph-mon[27511]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:50.205 INFO:teuthology.orchestra.run.smithi069.stderr:Inferring config /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/mon.smithi069/config 2024-06-11T13:41:50.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:50 smithi069 ceph-mon[25054]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:41:50.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:41:50.960 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:41:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:50 smithi135 ceph-mon[27511]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:41:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:41:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:41:51.943 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:51 smithi069 ceph-mon[25054]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:51.944 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:41:51.944 INFO:teuthology.orchestra.run.smithi069.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-06-11T13:41:52.045 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:51 smithi135 ceph-mon[27511]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:52.557 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-06-11T13:41:52.557 INFO:tasks.cephadm:Setup complete, yielding 2024-06-11T13:41:52.557 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-11T13:41:52.567 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:41:52.567 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch status' 2024-06-11T13:41:52.873 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:52 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3948042310' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-11T13:41:53.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:52 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3948042310' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-06-11T13:41:53.797 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:53 smithi069 ceph-mon[25054]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:54.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:53 smithi135 ceph-mon[27511]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:54.441 INFO:teuthology.orchestra.run.smithi069.stdout:Backend: cephadm 2024-06-11T13:41:54.442 INFO:teuthology.orchestra.run.smithi069.stdout:Available: Yes 2024-06-11T13:41:54.442 INFO:teuthology.orchestra.run.smithi069.stdout:Paused: No 2024-06-11T13:41:54.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:54 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:41:54.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:54 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:41:54.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:54 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:41:54.880 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:54 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:41:54.880 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:54 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:41:54.880 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:54 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:41:55.103 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch ps' 2024-06-11T13:41:55.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:55 smithi069 ceph-mon[25054]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:55.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:55 smithi069 ceph-mon[25054]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:41:55.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:55 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:41:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:55 smithi135 ceph-mon[27511]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:55 smithi135 ceph-mon[27511]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:41:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:55 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c 59bb0c633338 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (4m) 65s ago 4m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (3m) 2m ago 5m 45.7M - 8.3.5 dad864ee21e9 c3e2fd2cea97 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 15636a05991a 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283 running (4m) 65s ago 4m 413M - 17.2.0 e1d6a67b021e 7ad95b642fed 2024-06-11T13:41:56.974 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (6m) 2m ago 7m 45.1M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (4m) 65s ago 4m 42.1M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (5m) 2m ago 5m 17.7M - 1dbe0e931976 2b3eb7456a5b 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (4m) 65s ago 4m 19.1M - 1dbe0e931976 3b824b2b7502 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (4m) 2m ago 4m 34.1M 3216M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (3m) 2m ago 3m 36.0M 3216M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (3m) 2m ago 3m 31.7M 3216M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (2m) 2m ago 2m 25.9M 3216M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (2m) 65s ago 2m 36.2M 3984M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (119s) 65s ago 119s 35.2M 3984M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (95s) 65s ago 94s 33.1M 3984M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (70s) 65s ago 70s 13.0M 3984M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:41:56.975 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (4m) 2m ago 4m 52.7M - 514e6a882f6e ccd8f2b5541e 2024-06-11T13:41:57.545 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch ls' 2024-06-11T13:41:58.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:57 smithi069 ceph-mon[25054]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:58.170 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:57 smithi069 ceph-mon[25054]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:41:58.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:57 smithi135 ceph-mon[27511]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:41:58.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:57 smithi135 ceph-mon[27511]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:41:59.488 INFO:teuthology.orchestra.run.smithi069.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:crash 2/2 2m ago 6m * 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:mgr 2/2 2m ago 6m count:2 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:mon 2/2 2m ago 5m smithi069:172.21.15.69=smithi069;smithi135:172.21.15.135=smithi135;count:2 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:osd 8 2m ago - 2024-06-11T13:41:59.489 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-06-11T13:42:00.054 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:41:59 smithi069 ceph-mon[25054]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:00.079 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch host ls' 2024-06-11T13:42:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:41:59 smithi135 ceph-mon[27511]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:00.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:00 smithi069 ceph-mon[25054]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:01.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:00 smithi135 ceph-mon[27511]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:02.007 INFO:teuthology.orchestra.run.smithi069.stdout:HOST ADDR LABELS STATUS 2024-06-11T13:42:02.007 INFO:teuthology.orchestra.run.smithi069.stdout:smithi069 172.21.15.69 2024-06-11T13:42:02.007 INFO:teuthology.orchestra.run.smithi069.stdout:smithi135 172.21.15.135 2024-06-11T13:42:02.007 INFO:teuthology.orchestra.run.smithi069.stdout:2 hosts in cluster 2024-06-11T13:42:02.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:01 smithi069 ceph-mon[25054]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:02.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:01 smithi135 ceph-mon[27511]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:02.633 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph orch device ls' 2024-06-11T13:42:03.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:03 smithi069 ceph-mon[25054]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:03.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:03 smithi069 ceph-mon[25054]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:04.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:03 smithi135 ceph-mon[27511]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:04.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:03 smithi135 ceph-mon[27511]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:04.557 INFO:teuthology.orchestra.run.smithi069.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-06-11T13:42:04.558 INFO:teuthology.orchestra.run.smithi069.stdout:smithi069 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400Y0400BGN 400G LVM detected, locked 2024-06-11T13:42:04.558 INFO:teuthology.orchestra.run.smithi069.stdout:smithi135 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620401BA400BGN 400G LVM detected, locked 2024-06-11T13:42:05.186 INFO:teuthology.run_tasks:Running task vip.exec... 2024-06-11T13:42:05.262 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:42:05.263 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-06-11T13:42:05.297 INFO:teuthology.orchestra.run.smithi069.stderr:+ systemctl stop nfs-server 2024-06-11T13:42:05.304 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi135.front.sepia.ceph.com 2024-06-11T13:42:05.305 DEBUG:teuthology.orchestra.run.smithi135:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-06-11T13:42:05.337 INFO:teuthology.orchestra.run.smithi135.stderr:+ systemctl stop nfs-server 2024-06-11T13:42:05.345 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-11T13:42:05.445 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:42:05.445 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-06-11T13:42:06.075 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:05 smithi069 ceph-mon[25054]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:06.076 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:05 smithi069 ceph-mon[25054]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:05 smithi135 ceph-mon[27511]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:05 smithi135 ceph-mon[27511]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:08.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:08 smithi069 ceph-mon[25054]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:08.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:08 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-06-11T13:42:08.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:08 smithi135 ceph-mon[27511]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:08.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:08 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:08 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[25050]: 2024-06-11T13:42:08.918+0000 7f0aa3a5f700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: osdmap e61: 8 total, 8 up, 8 in 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: pgmap v208: 33 pgs: 6 creating+peering, 26 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:09.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: osdmap e62: 8 total, 8 up, 8 in 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: osdmap e63: 8 total, 8 up, 8 in 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: fsmap foofs:0 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: Saving service mds.foofs spec with placement count:2 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:09.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:09.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:09.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T13:42:09.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-11T13:42:09.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:09.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:09 smithi069 ceph-mon[25054]: Deploying daemon mds.foofs.smithi069.nsgdqs on smithi069 2024-06-11T13:42:09.448 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:09.448 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-06-11T13:42:09.448 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: osdmap e61: 8 total, 8 up, 8 in 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: pgmap v208: 33 pgs: 6 creating+peering, 26 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: osdmap e62: 8 total, 8 up, 8 in 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-06-11T13:42:09.449 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: osdmap e63: 8 total, 8 up, 8 in 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: fsmap foofs:0 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: Saving service mds.foofs spec with placement count:2 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T13:42:09.450 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-11T13:42:09.451 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:09.451 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:09 smithi135 ceph-mon[27511]: Deploying daemon mds.foofs.smithi069.nsgdqs on smithi069 2024-06-11T13:42:09.691 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-06-11T13:42:09.701 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-06-11T13:42:09.701 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-11T13:42:11.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:10 smithi069 ceph-mon[25054]: osdmap e64: 8 total, 8 up, 8 in 2024-06-11T13:42:11.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:10 smithi135 ceph-mon[27511]: osdmap e64: 8 total, 8 up, 8 in 2024-06-11T13:42:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:11 smithi069 ceph-mon[25054]: pgmap v212: 65 pgs: 1 creating+activating, 26 creating+peering, 16 unknown, 22 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:11.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:11 smithi069 ceph-mon[25054]: osdmap e65: 8 total, 8 up, 8 in 2024-06-11T13:42:12.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:11 smithi135 ceph-mon[27511]: pgmap v212: 65 pgs: 1 creating+activating, 26 creating+peering, 16 unknown, 22 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:12.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:11 smithi135 ceph-mon[27511]: osdmap e65: 8 total, 8 up, 8 in 2024-06-11T13:42:12.588 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:42:12.588 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-11T13:35:47.061980Z", "last_refresh": "2024-06-11T13:39:26.613724Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-11T13:35:41.037050Z", "last_refresh": "2024-06-11T13:39:26.613790Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-11T13:35:43.843322Z", "last_refresh": "2024-06-11T13:39:26.613854Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-11T13:42:08.940574Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-11T13:42:08.936725Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-11T13:35:38.798707Z", "last_refresh": "2024-06-11T13:39:26.613652Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi135:172.21.15.135=smithi135"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-11T13:36:28.693457Z", "last_refresh": "2024-06-11T13:39:26.613498Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-11T13:35:45.126603Z", "last_refresh": "2024-06-11T13:39:26.613915Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-11T13:39:26.614051Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-11T13:35:42.191099Z", "last_refresh": "2024-06-11T13:39:26.613988Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-11T13:42:13.315 INFO:tasks.cephadm:mds.foofs has 0/2 2024-06-11T13:42:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: pgmap v214: 65 pgs: 1 creating+activating, 26 creating+peering, 11 unknown, 27 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T13:42:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: Deploying daemon mds.foofs.smithi135.wtrawd on smithi135 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: mds.? [v2:172.21.15.69:6834/2689424606,v1:172.21.15.69:6835/2689424606] up:boot 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: daemon mds.foofs.smithi069.nsgdqs assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: Cluster is now healthy 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: fsmap foofs:0 1 up:standby 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:creating} 2024-06-11T13:42:14.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:13 smithi135 ceph-mon[27511]: daemon mds.foofs.smithi069.nsgdqs is now active in filesystem foofs as rank 0 2024-06-11T13:42:14.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: pgmap v214: 65 pgs: 1 creating+activating, 26 creating+peering, 11 unknown, 27 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: Deploying daemon mds.foofs.smithi135.wtrawd on smithi135 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: mds.? [v2:172.21.15.69:6834/2689424606,v1:172.21.15.69:6835/2689424606] up:boot 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: daemon mds.foofs.smithi069.nsgdqs assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-06-11T13:42:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: Cluster is now healthy 2024-06-11T13:42:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: fsmap foofs:0 1 up:standby 2024-06-11T13:42:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:42:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:creating} 2024-06-11T13:42:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:13 smithi069 ceph-mon[25054]: daemon mds.foofs.smithi069.nsgdqs is now active in filesystem foofs as rank 0 2024-06-11T13:42:14.315 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-11T13:42:15.014 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:14 smithi135 ceph-mon[27511]: mds.? [v2:172.21.15.69:6834/2689424606,v1:172.21.15.69:6835/2689424606] up:active 2024-06-11T13:42:15.014 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:14 smithi135 ceph-mon[27511]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 2024-06-11T13:42:15.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:14 smithi069 ceph-mon[25054]: mds.? [v2:172.21.15.69:6834/2689424606,v1:172.21.15.69:6835/2689424606] up:active 2024-06-11T13:42:15.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:14 smithi069 ceph-mon[25054]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 2024-06-11T13:42:16.244 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:42:16.244 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-11T13:35:47.061980Z", "last_refresh": "2024-06-11T13:39:26.613724Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-11T13:35:41.037050Z", "last_refresh": "2024-06-11T13:39:26.613790Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-11T13:35:43.843322Z", "last_refresh": "2024-06-11T13:39:26.613854Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-11T13:42:08.940574Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-11T13:42:08.936725Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-11T13:35:38.798707Z", "last_refresh": "2024-06-11T13:39:26.613652Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi135:172.21.15.135=smithi135"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-11T13:36:28.693457Z", "last_refresh": "2024-06-11T13:39:26.613498Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-11T13:35:45.126603Z", "last_refresh": "2024-06-11T13:39:26.613915Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-11T13:39:26.614051Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-11T13:35:42.191099Z", "last_refresh": "2024-06-11T13:39:26.613988Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-11T13:42:16.244 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: pgmap v215: 65 pgs: 1 creating+activating, 11 creating+peering, 53 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-06-11T13:42:16.244 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:16.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:16.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:16.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:16.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:boot 2024-06-11T13:42:16.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T13:42:16.245 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:15 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:42:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: pgmap v215: 65 pgs: 1 creating+activating, 11 creating+peering, 53 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-06-11T13:42:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:16.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:16.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:boot 2024-06-11T13:42:16.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T13:42:16.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:15 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:42:17.314 INFO:tasks.cephadm:mds.foofs has 0/2 2024-06-11T13:42:18.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:17 smithi069 ceph-mon[25054]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s wr, 6 op/s 2024-06-11T13:42:18.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:17 smithi069 ceph-mon[25054]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:18.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:17 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:17 smithi135 ceph-mon[27511]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s wr, 6 op/s 2024-06-11T13:42:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:17 smithi135 ceph-mon[27511]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:17 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:18.315 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-11T13:42:20.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:19 smithi069 ceph-mon[25054]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-06-11T13:42:20.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:19 smithi135 ceph-mon[27511]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-06-11T13:42:21.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:21 smithi069 ceph-mon[25054]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-06-11T13:42:21.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:21 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:21.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:21 smithi135 ceph-mon[27511]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-06-11T13:42:21.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:21 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:21.497 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:42:21.497 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-11T13:35:47.061980Z", "last_refresh": "2024-06-11T13:39:26.613724Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-11T13:35:41.037050Z", "last_refresh": "2024-06-11T13:39:26.613790Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-11T13:35:43.843322Z", "last_refresh": "2024-06-11T13:39:26.613854Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-11T13:42:08.940574Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-11T13:42:08.936725Z", "last_refresh": "2024-06-11T13:42:17.139244Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-11T13:35:38.798707Z", "last_refresh": "2024-06-11T13:39:26.613652Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi135:172.21.15.135=smithi135"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-11T13:36:28.693457Z", "last_refresh": "2024-06-11T13:39:26.613498Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-11T13:35:45.126603Z", "last_refresh": "2024-06-11T13:39:26.613915Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-11T13:39:26.614051Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-11T13:35:42.191099Z", "last_refresh": "2024-06-11T13:39:26.613988Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-11T13:42:22.246 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:22 smithi069 ceph-mon[25054]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:22.274 INFO:tasks.cephadm:mds.foofs has 1/2 2024-06-11T13:42:22.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:22 smithi135 ceph-mon[27511]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:23.275 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-11T13:42:23.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-06-11T13:42:23.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:23.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:23.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:23.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:23.585 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:23.586 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:23 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:23.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:23 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:25.370 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:42:25.370 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-11T13:35:47.061980Z", "last_refresh": "2024-06-11T13:42:22.535576Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-11T13:35:41.037050Z", "last_refresh": "2024-06-11T13:42:17.138604Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-06-11T13:35:43.843322Z", "last_refresh": "2024-06-11T13:42:22.535702Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-06-11T13:42:08.940574Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-11T13:42:08.936725Z", "last_refresh": "2024-06-11T13:42:17.139244Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-11T13:35:38.798707Z", "last_refresh": "2024-06-11T13:42:17.138760Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi135:172.21.15.135=smithi135"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-11T13:36:28.693457Z", "last_refresh": "2024-06-11T13:42:17.138828Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-11T13:35:45.126603Z", "last_refresh": "2024-06-11T13:42:17.138890Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-06-11T13:42:17.138979Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-11T13:35:42.191099Z", "last_refresh": "2024-06-11T13:42:22.535824Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-11T13:42:25.725 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:25 smithi069 ceph-mon[25054]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-06-11T13:42:25.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:25 smithi135 ceph-mon[27511]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-06-11T13:42:26.032 INFO:tasks.cephadm:mds.foofs has 2/2 2024-06-11T13:42:26.033 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-11T13:42:26.042 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:42:26.043 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-06-11T13:42:26.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:26 smithi069 ceph-mon[25054]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:26.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:26 smithi135 ceph-mon[27511]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T13:42:27.616 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:27 smithi069 ceph-mon[25054]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-06-11T13:42:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:27 smithi135 ceph-mon[27511]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-06-11T13:42:28.995 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-06-11T13:42:29.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-06-11T13:42:29.612 INFO:teuthology.orchestra.run.smithi069.stdout:NFS Cluster Created Successfully 2024-06-11T13:42:29.837 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:29 smithi069 ceph-mon[25054]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-06-11T13:42:29.837 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:29 smithi069 ceph-mon[25054]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:29.838 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-06-11T13:42:29.838 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:29 smithi069 ceph-mon[25054]: osdmap e66: 8 total, 8 up, 8 in 2024-06-11T13:42:29.838 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-06-11T13:42:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:29 smithi135 ceph-mon[27511]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-06-11T13:42:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:29 smithi135 ceph-mon[27511]: from='client.14516 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-06-11T13:42:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:29 smithi135 ceph-mon[27511]: osdmap e66: 8 total, 8 up, 8 in 2024-06-11T13:42:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-06-11T13:42:30.371 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-06-11T13:42:30.631 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: osdmap e67: 8 total, 8 up, 8 in 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: Saving service nfs.foo spec with placement count:2 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-11T13:42:30.632 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: Rados config object exists: conf-nfs.foo 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg-rgw 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:30.633 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:30 smithi069 ceph-mon[25054]: Deploying daemon nfs.foo.0.0.smithi069.txqrqg on smithi069 2024-06-11T13:42:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-06-11T13:42:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: osdmap e67: 8 total, 8 up, 8 in 2024-06-11T13:42:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: Saving service nfs.foo spec with placement count:2 2024-06-11T13:42:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T13:42:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: Rados config object exists: conf-nfs.foo 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg-rgw 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:31.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:30 smithi135 ceph-mon[27511]: Deploying daemon nfs.foo.0.0.smithi069.txqrqg on smithi069 2024-06-11T13:42:31.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:31 smithi069 ceph-mon[25054]: pgmap v225: 97 pgs: 11 creating+peering, 9 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T13:42:31.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:31 smithi069 ceph-mon[25054]: osdmap e68: 8 total, 8 up, 8 in 2024-06-11T13:42:31.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:31 smithi069 ceph-mon[25054]: mgrmap e20: smithi069.iruzyw(active, since 5m), standbys: smithi135.xcould 2024-06-11T13:42:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:31 smithi135 ceph-mon[27511]: pgmap v225: 97 pgs: 11 creating+peering, 9 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T13:42:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:31 smithi135 ceph-mon[27511]: osdmap e68: 8 total, 8 up, 8 in 2024-06-11T13:42:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:31 smithi135 ceph-mon[27511]: mgrmap e20: smithi069.iruzyw(active, since 5m), standbys: smithi135.xcould 2024-06-11T13:42:33.691 INFO:teuthology.orchestra.run.smithi069.stderr:Invalid command: Unexpected argument '--clusterid' 2024-06-11T13:42:33.691 INFO:teuthology.orchestra.run.smithi069.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-06-11T13:42:33.691 INFO:teuthology.orchestra.run.smithi069.stderr:Error EINVAL: invalid command 2024-06-11T13:42:33.860 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:33 smithi069 ceph-mon[25054]: pgmap v227: 97 pgs: 11 creating+peering, 86 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-06-11T13:42:34.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:33 smithi135 ceph-mon[27511]: pgmap v227: 97 pgs: 11 creating+peering, 86 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-06-11T13:42:34.049 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:42:34.049 INFO:teuthology.orchestra.run.smithi069.stdout: "bind": "/fake", 2024-06-11T13:42:34.049 INFO:teuthology.orchestra.run.smithi069.stdout: "fs": "foofs", 2024-06-11T13:42:34.049 INFO:teuthology.orchestra.run.smithi069.stdout: "path": "/", 2024-06-11T13:42:34.049 INFO:teuthology.orchestra.run.smithi069.stdout: "cluster": "foo", 2024-06-11T13:42:34.049 INFO:teuthology.orchestra.run.smithi069.stdout: "mode": "RW" 2024-06-11T13:42:34.050 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:42:34.737 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='client.14538 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: pgmap v228: 97 pgs: 8 creating+peering, 89 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: Creating key for client.nfs.foo.1.0.smithi135.mndnps 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-11T13:42:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-11T13:42:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T13:42:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-06-11T13:42:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T13:42:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:35 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-06-11T13:42:35.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='client.14538 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:35.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: pgmap v228: 97 pgs: 8 creating+peering, 89 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-06-11T13:42:35.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:35.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: Creating key for client.nfs.foo.1.0.smithi135.mndnps 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:35.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:35 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-06-11T13:42:37.630 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: Rados config object exists: conf-nfs.foo 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: Creating key for client.nfs.foo.1.0.smithi135.mndnps-rgw 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:37.631 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:37 smithi135 ceph-mon[27511]: Deploying daemon nfs.foo.1.0.smithi135.mndnps on smithi135 2024-06-11T13:42:37.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: Rados config object exists: conf-nfs.foo 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: Creating key for client.nfs.foo.1.0.smithi135.mndnps-rgw 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:37 smithi069 ceph-mon[25054]: Deploying daemon nfs.foo.1.0.smithi135.mndnps on smithi135 2024-06-11T13:42:38.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:38 smithi135 ceph-mon[27511]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:38.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:38 smithi069 ceph-mon[25054]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:39 smithi135 ceph-mon[27511]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-11T13:42:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:39.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:39.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:39 smithi069 ceph-mon[25054]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-06-11T13:42:39.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:39.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:39.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:39.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:40.847 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:40 smithi135 ceph-mon[27511]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:40.847 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:40 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:40.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:40 smithi069 ceph-mon[25054]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:40.955 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:40 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:41.839 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:41 smithi135 ceph-mon[27511]: pgmap v231: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-06-11T13:42:41.839 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:41 smithi135 ceph-mon[27511]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:41.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:41 smithi069 ceph-mon[25054]: pgmap v231: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-06-11T13:42:41.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:41 smithi069 ceph-mon[25054]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:42.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:42 smithi069 ceph-mon[25054]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:42.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:42 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:42 smithi135 ceph-mon[27511]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:42 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:43.962 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:43 smithi069 ceph-mon[25054]: pgmap v232: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-06-11T13:42:44.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:43 smithi135 ceph-mon[27511]: pgmap v232: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-06-11T13:42:44.526 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-06-11T13:42:44.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='client.14588 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:42:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:45.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:42:45.222 INFO:teuthology.run_tasks:Running task vip.exec... 2024-06-11T13:42:45.234 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:42:45.234 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-06-11T13:42:45.261 INFO:teuthology.orchestra.run.smithi069.stderr:+ mkdir /mnt/foo 2024-06-11T13:42:45.267 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-06-11T13:42:45.331 INFO:teuthology.orchestra.run.smithi069.stderr:++ hostname 2024-06-11T13:42:45.332 INFO:teuthology.orchestra.run.smithi069.stderr:+ mount -t nfs smithi069:/fake /mnt/foo -o sync 2024-06-11T13:42:45.762 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-06-11T13:42:45.827 INFO:teuthology.orchestra.run.smithi069.stderr:+ echo test 2024-06-11T13:42:45.835 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:45 smithi069 ceph-mon[25054]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.7 KiB/s wr, 5 op/s 2024-06-11T13:42:45.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:45 smithi069 ceph-mon[25054]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:45.837 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-06-11T13:42:45.900 INFO:teuthology.orchestra.run.smithi069.stderr:+ sync 2024-06-11T13:42:46.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:45 smithi135 ceph-mon[27511]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.7 KiB/s wr, 5 op/s 2024-06-11T13:42:46.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:45 smithi135 ceph-mon[27511]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:42:46.080 INFO:teuthology.run_tasks:Running task parallel... 2024-06-11T13:42:46.090 INFO:teuthology.task.parallel:starting parallel... 2024-06-11T13:42:46.091 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-06-11T13:42:46.091 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-06-11T13:42:46.092 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:42:46.092 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-06-11T13:42:46.092 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-06-11T13:42:46.093 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-06-11T13:42:46.093 INFO:teuthology.task.exec:Executing custom commands... 2024-06-11T13:42:46.093 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:42:46.094 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-06-11T13:42:46.177 INFO:teuthology.orchestra.run.smithi069.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-06-11T13:42:46.177 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:42:46.177 INFO:teuthology.orchestra.run.smithi069.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-06-11T13:42:46.179 INFO:teuthology.orchestra.run.smithi069.stdout:3 of 5 processes prepared for launch 0 sec 2024-06-11T13:42:46.279 INFO:teuthology.orchestra.run.smithi069.stdout:5 of 5 processes prepared for launch 0 sec 2024-06-11T13:42:46.279 INFO:teuthology.orchestra.run.smithi069.stdout:releasing clients 2024-06-11T13:42:47.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 316 57.62 MB/sec warmup 1 sec latency 31.281 ms 2024-06-11T13:42:47.679 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:47 smithi069 ceph-mon[25054]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-06-11T13:42:48.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:47 smithi135 ceph-mon[27511]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-06-11T13:42:48.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 680 52.93 MB/sec warmup 2 sec latency 11.466 ms 2024-06-11T13:42:49.279 INFO:teuthology.orchestra.run.smithi069.stdout: 5 1569 43.54 MB/sec warmup 3 sec latency 37.956 ms 2024-06-11T13:42:49.693 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:49 smithi069 ceph-mon[25054]: pgmap v235: 97 pgs: 97 active+clean; 18 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.5 MiB/s wr, 32 op/s 2024-06-11T13:42:50.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:49 smithi135 ceph-mon[27511]: pgmap v235: 97 pgs: 97 active+clean; 18 MiB data, 57 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.5 MiB/s wr, 32 op/s 2024-06-11T13:42:50.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 2650 37.70 MB/sec warmup 4 sec latency 16.934 ms 2024-06-11T13:42:50.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]: dispatch 2024-06-11T13:42:50.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2024-06-11T13:42:50.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-06-11T13:42:50.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-11T13:42:50.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:42:50.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:42:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]: dispatch 2024-06-11T13:42:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2024-06-11T13:42:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-06-11T13:42:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-06-11T13:42:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:42:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:42:51.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 3782 39.62 MB/sec warmup 5 sec latency 14.855 ms 2024-06-11T13:42:51.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:51 smithi069 ceph-mon[25054]: pgmap v236: 97 pgs: 97 active+clean; 95 MiB data, 180 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 8.5 MiB/s wr, 423 op/s 2024-06-11T13:42:51.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]': finished 2024-06-11T13:42:51.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-06-11T13:42:51.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-06-11T13:42:51.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-06-11T13:42:51.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:51 smithi069 ceph-mon[25054]: osdmap e69: 8 total, 8 up, 8 in 2024-06-11T13:42:51.921 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:51 smithi135 ceph-mon[27511]: pgmap v236: 97 pgs: 97 active+clean; 95 MiB data, 180 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 8.5 MiB/s wr, 423 op/s 2024-06-11T13:42:51.921 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1c", "id": [7, 4]}]': finished 2024-06-11T13:42:51.921 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2024-06-11T13:42:51.922 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-06-11T13:42:51.922 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-06-11T13:42:51.922 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:51 smithi135 ceph-mon[27511]: osdmap e69: 8 total, 8 up, 8 in 2024-06-11T13:42:52.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 4588 34.64 MB/sec warmup 6 sec latency 17.724 ms 2024-06-11T13:42:52.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:52 smithi069 ceph-mon[25054]: osdmap e70: 8 total, 8 up, 8 in 2024-06-11T13:42:53.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:52 smithi135 ceph-mon[27511]: osdmap e70: 8 total, 8 up, 8 in 2024-06-11T13:42:53.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 5725 32.82 MB/sec warmup 7 sec latency 15.576 ms 2024-06-11T13:42:53.718 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:53 smithi069 ceph-mon[25054]: pgmap v239: 97 pgs: 97 active+clean; 107 MiB data, 196 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 15 MiB/s wr, 675 op/s 2024-06-11T13:42:54.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:53 smithi135 ceph-mon[27511]: pgmap v239: 97 pgs: 97 active+clean; 107 MiB data, 196 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 15 MiB/s wr, 675 op/s 2024-06-11T13:42:54.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 6704 33.39 MB/sec warmup 8 sec latency 13.396 ms 2024-06-11T13:42:55.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 7586 32.88 MB/sec warmup 9 sec latency 13.070 ms 2024-06-11T13:42:55.727 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:55 smithi069 ceph-mon[25054]: pgmap v240: 97 pgs: 97 active+clean; 120 MiB data, 245 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 18 MiB/s wr, 945 op/s 2024-06-11T13:42:56.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:55 smithi135 ceph-mon[27511]: pgmap v240: 97 pgs: 97 active+clean; 120 MiB data, 245 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 18 MiB/s wr, 945 op/s 2024-06-11T13:42:56.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 8559 31.86 MB/sec warmup 10 sec latency 16.119 ms 2024-06-11T13:42:57.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 9567 30.28 MB/sec warmup 11 sec latency 21.470 ms 2024-06-11T13:42:57.783 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:57 smithi069 ceph-mon[25054]: pgmap v241: 97 pgs: 97 active+clean; 137 MiB data, 293 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 MiB/s wr, 1.49k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:42:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:57 smithi135 ceph-mon[27511]: pgmap v241: 97 pgs: 97 active+clean; 137 MiB data, 293 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 MiB/s wr, 1.49k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:42:58.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 10765 32.13 MB/sec warmup 12 sec latency 18.555 ms 2024-06-11T13:42:59.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 11415 30.27 MB/sec warmup 13 sec latency 25.265 ms 2024-06-11T13:42:59.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:42:59 smithi069 ceph-mon[25054]: pgmap v242: 97 pgs: 97 active+clean; 148 MiB data, 320 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 15 MiB/s wr, 1.30k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:42:59 smithi135 ceph-mon[27511]: pgmap v242: 97 pgs: 97 active+clean; 148 MiB data, 320 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 15 MiB/s wr, 1.30k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:00.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 12421 29.87 MB/sec warmup 14 sec latency 14.783 ms 2024-06-11T13:43:01.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 13489 29.87 MB/sec warmup 15 sec latency 18.452 ms 2024-06-11T13:43:01.805 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:01 smithi069 ceph-mon[25054]: pgmap v243: 97 pgs: 97 active+clean; 151 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 1.32k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:01 smithi135 ceph-mon[27511]: pgmap v243: 97 pgs: 97 active+clean; 151 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 1.32k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:02.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 14459 30.35 MB/sec warmup 16 sec latency 15.562 ms 2024-06-11T13:43:03.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 15115 28.88 MB/sec warmup 17 sec latency 16.882 ms 2024-06-11T13:43:03.832 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:03 smithi069 ceph-mon[25054]: pgmap v244: 97 pgs: 97 active+clean; 152 MiB data, 432 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 MiB/s wr, 1.23k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:04.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:03 smithi135 ceph-mon[27511]: pgmap v244: 97 pgs: 97 active+clean; 152 MiB data, 432 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 14 MiB/s wr, 1.23k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:04.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 16019 28.56 MB/sec warmup 18 sec latency 16.666 ms 2024-06-11T13:43:05.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 17225 29.04 MB/sec warmup 19 sec latency 10.400 ms 2024-06-11T13:43:05.842 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:05 smithi069 ceph-mon[25054]: pgmap v245: 97 pgs: 97 active+clean; 160 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 14 MiB/s wr, 1.34k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:06.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:05 smithi135 ceph-mon[27511]: pgmap v245: 97 pgs: 97 active+clean; 160 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 14 MiB/s wr, 1.34k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:06.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 18150 29.14 MB/sec warmup 20 sec latency 19.137 ms 2024-06-11T13:43:07.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 18939 28.38 MB/sec warmup 21 sec latency 15.862 ms 2024-06-11T13:43:07.854 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:07 smithi069 ceph-mon[25054]: pgmap v246: 97 pgs: 97 active+clean; 161 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 15 MiB/s wr, 1.38k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:07 smithi135 ceph-mon[27511]: pgmap v246: 97 pgs: 97 active+clean; 161 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 15 MiB/s wr, 1.38k op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-06-11T13:43:08.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 19972 27.89 MB/sec warmup 22 sec latency 17.516 ms 2024-06-11T13:43:09.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 21048 28.48 MB/sec warmup 23 sec latency 16.010 ms 2024-06-11T13:43:09.820 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:09 smithi069 ceph-mon[25054]: pgmap v247: 97 pgs: 97 active+clean; 168 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-06-11T13:43:10.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:09 smithi135 ceph-mon[27511]: pgmap v247: 97 pgs: 97 active+clean; 168 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.10k op/s 2024-06-11T13:43:10.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 21846 28.29 MB/sec warmup 24 sec latency 20.789 ms 2024-06-11T13:43:11.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 22685 27.92 MB/sec warmup 25 sec latency 15.716 ms 2024-06-11T13:43:11.831 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:11 smithi069 ceph-mon[25054]: pgmap v248: 97 pgs: 97 active+clean; 181 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 14 MiB/s wr, 1.21k op/s 2024-06-11T13:43:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:11 smithi135 ceph-mon[27511]: pgmap v248: 97 pgs: 97 active+clean; 181 MiB data, 648 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 14 MiB/s wr, 1.21k op/s 2024-06-11T13:43:12.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 23651 27.38 MB/sec warmup 26 sec latency 22.584 ms 2024-06-11T13:43:13.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 24873 28.28 MB/sec warmup 27 sec latency 12.570 ms 2024-06-11T13:43:13.842 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:13 smithi069 ceph-mon[25054]: pgmap v249: 97 pgs: 97 active+clean; 182 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.03k op/s 2024-06-11T13:43:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:13 smithi135 ceph-mon[27511]: pgmap v249: 97 pgs: 97 active+clean; 182 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.03k op/s 2024-06-11T13:43:14.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 25589 27.70 MB/sec warmup 28 sec latency 28.326 ms 2024-06-11T13:43:15.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 26510 27.60 MB/sec warmup 29 sec latency 13.692 ms 2024-06-11T13:43:15.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:15 smithi069 ceph-mon[25054]: pgmap v250: 97 pgs: 97 active+clean; 191 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-06-11T13:43:16.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:15 smithi135 ceph-mon[27511]: pgmap v250: 97 pgs: 97 active+clean; 191 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-06-11T13:43:16.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 27526 27.29 MB/sec warmup 30 sec latency 18.911 ms 2024-06-11T13:43:17.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 28611 27.91 MB/sec warmup 31 sec latency 19.285 ms 2024-06-11T13:43:17.860 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:17 smithi069 ceph-mon[25054]: pgmap v251: 97 pgs: 97 active+clean; 198 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.27k op/s 2024-06-11T13:43:18.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:17 smithi135 ceph-mon[27511]: pgmap v251: 97 pgs: 97 active+clean; 198 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 15 MiB/s wr, 1.27k op/s 2024-06-11T13:43:18.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 29359 27.33 MB/sec warmup 32 sec latency 13.741 ms 2024-06-11T13:43:19.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 30379 27.22 MB/sec warmup 33 sec latency 21.592 ms 2024-06-11T13:43:19.874 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:19 smithi069 ceph-mon[25054]: pgmap v252: 97 pgs: 97 active+clean; 201 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-06-11T13:43:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:19 smithi135 ceph-mon[27511]: pgmap v252: 97 pgs: 97 active+clean; 201 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-06-11T13:43:20.280 INFO:teuthology.orchestra.run.smithi069.stdout: 5 31410 27.43 MB/sec warmup 34 sec latency 17.624 ms 2024-06-11T13:43:21.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 32312 27.55 MB/sec warmup 35 sec latency 19.508 ms 2024-06-11T13:43:21.637 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:21 smithi069 ceph-mon[25054]: pgmap v253: 97 pgs: 97 active+clean; 204 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.34k op/s 2024-06-11T13:43:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:21 smithi135 ceph-mon[27511]: pgmap v253: 97 pgs: 97 active+clean; 204 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.34k op/s 2024-06-11T13:43:22.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 33062 27.11 MB/sec warmup 36 sec latency 21.546 ms 2024-06-11T13:43:23.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 33994 26.87 MB/sec warmup 37 sec latency 26.050 ms 2024-06-11T13:43:23.659 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:23 smithi069 ceph-mon[25054]: pgmap v254: 97 pgs: 97 active+clean; 206 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-06-11T13:43:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:23 smithi135 ceph-mon[27511]: pgmap v254: 97 pgs: 97 active+clean; 206 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-06-11T13:43:24.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 35154 27.23 MB/sec warmup 38 sec latency 14.117 ms 2024-06-11T13:43:25.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 36034 27.22 MB/sec warmup 39 sec latency 14.671 ms 2024-06-11T13:43:25.668 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:25 smithi069 ceph-mon[25054]: pgmap v255: 97 pgs: 97 active+clean; 212 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.24k op/s 2024-06-11T13:43:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:25 smithi135 ceph-mon[27511]: pgmap v255: 97 pgs: 97 active+clean; 212 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.24k op/s 2024-06-11T13:43:26.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 36810 26.87 MB/sec warmup 40 sec latency 68.069 ms 2024-06-11T13:43:27.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 37821 26.71 MB/sec warmup 41 sec latency 21.689 ms 2024-06-11T13:43:27.676 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:27 smithi069 ceph-mon[25054]: pgmap v256: 97 pgs: 97 active+clean; 226 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 14 MiB/s wr, 1.26k op/s 2024-06-11T13:43:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:27 smithi135 ceph-mon[27511]: pgmap v256: 97 pgs: 97 active+clean; 226 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 14 MiB/s wr, 1.26k op/s 2024-06-11T13:43:28.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 38645 26.88 MB/sec warmup 42 sec latency 47.974 ms 2024-06-11T13:43:29.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 39438 26.88 MB/sec warmup 43 sec latency 17.793 ms 2024-06-11T13:43:29.686 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:29 smithi069 ceph-mon[25054]: pgmap v257: 97 pgs: 97 active+clean; 228 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.10k op/s 2024-06-11T13:43:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:29 smithi135 ceph-mon[27511]: pgmap v257: 97 pgs: 97 active+clean; 228 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 13 MiB/s wr, 1.10k op/s 2024-06-11T13:43:30.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 40124 26.48 MB/sec warmup 44 sec latency 24.140 ms 2024-06-11T13:43:31.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 40990 26.35 MB/sec warmup 45 sec latency 22.990 ms 2024-06-11T13:43:31.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:31 smithi069 ceph-mon[25054]: pgmap v258: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.33k op/s 2024-06-11T13:43:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:31 smithi135 ceph-mon[27511]: pgmap v258: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.33k op/s 2024-06-11T13:43:32.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 41897 26.29 MB/sec warmup 46 sec latency 17.400 ms 2024-06-11T13:43:33.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 42733 26.51 MB/sec warmup 47 sec latency 25.242 ms 2024-06-11T13:43:33.707 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:33 smithi069 ceph-mon[25054]: pgmap v259: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.06k op/s 2024-06-11T13:43:34.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:33 smithi135 ceph-mon[27511]: pgmap v259: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.06k op/s 2024-06-11T13:43:34.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 43346 26.14 MB/sec warmup 48 sec latency 20.667 ms 2024-06-11T13:43:35.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 44216 26.05 MB/sec warmup 49 sec latency 22.392 ms 2024-06-11T13:43:35.730 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:35 smithi069 ceph-mon[25054]: pgmap v260: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-06-11T13:43:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:35 smithi135 ceph-mon[27511]: pgmap v260: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 13 MiB/s wr, 1.20k op/s 2024-06-11T13:43:36.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 45319 26.04 MB/sec warmup 50 sec latency 23.834 ms 2024-06-11T13:43:37.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 46231 26.21 MB/sec warmup 51 sec latency 23.449 ms 2024-06-11T13:43:37.738 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:37 smithi069 ceph-mon[25054]: pgmap v261: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.23k op/s 2024-06-11T13:43:38.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:37 smithi135 ceph-mon[27511]: pgmap v261: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.23k op/s 2024-06-11T13:43:38.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 46863 26.02 MB/sec warmup 52 sec latency 28.086 ms 2024-06-11T13:43:39.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 47700 25.80 MB/sec warmup 53 sec latency 18.889 ms 2024-06-11T13:43:39.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:39 smithi069 ceph-mon[25054]: pgmap v262: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-06-11T13:43:40.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:39 smithi135 ceph-mon[27511]: pgmap v262: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-06-11T13:43:40.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 48780 25.83 MB/sec warmup 54 sec latency 18.926 ms 2024-06-11T13:43:41.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 49784 26.04 MB/sec warmup 55 sec latency 22.449 ms 2024-06-11T13:43:41.757 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:41 smithi069 ceph-mon[25054]: pgmap v263: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 13 MiB/s wr, 1.28k op/s 2024-06-11T13:43:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:41 smithi135 ceph-mon[27511]: pgmap v263: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 13 MiB/s wr, 1.28k op/s 2024-06-11T13:43:42.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 50512 25.89 MB/sec warmup 56 sec latency 16.351 ms 2024-06-11T13:43:43.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 51433 25.77 MB/sec warmup 57 sec latency 25.678 ms 2024-06-11T13:43:43.767 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:43 smithi069 ceph-mon[25054]: pgmap v264: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 960 op/s 2024-06-11T13:43:44.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:43 smithi135 ceph-mon[27511]: pgmap v264: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 10 MiB/s wr, 960 op/s 2024-06-11T13:43:44.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 52465 25.78 MB/sec warmup 58 sec latency 21.149 ms 2024-06-11T13:43:44.694 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:43:44.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:43:44.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:44 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:43:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:43:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:43:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:44 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:43:45.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 53322 25.89 MB/sec warmup 59 sec latency 18.680 ms 2024-06-11T13:43:45.786 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:45 smithi069 ceph-mon[25054]: pgmap v265: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.17k op/s 2024-06-11T13:43:46.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:45 smithi135 ceph-mon[27511]: pgmap v265: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.17k op/s 2024-06-11T13:43:46.281 INFO:teuthology.orchestra.run.smithi069.stdout: 5 54031 25.75 MB/sec warmup 60 sec latency 19.182 ms 2024-06-11T13:43:47.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 54841 25.59 MB/sec warmup 61 sec latency 19.073 ms 2024-06-11T13:43:47.800 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:47 smithi069 ceph-mon[25054]: pgmap v266: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-06-11T13:43:48.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:47 smithi135 ceph-mon[27511]: pgmap v266: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 13 MiB/s wr, 1.16k op/s 2024-06-11T13:43:48.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 55727 25.48 MB/sec warmup 62 sec latency 21.779 ms 2024-06-11T13:43:49.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 56589 25.63 MB/sec warmup 63 sec latency 21.102 ms 2024-06-11T13:43:49.868 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:49 smithi069 ceph-mon[25054]: pgmap v267: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.06k op/s 2024-06-11T13:43:50.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:49 smithi135 ceph-mon[27511]: pgmap v267: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 12 MiB/s wr, 1.06k op/s 2024-06-11T13:43:50.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 57304 25.56 MB/sec warmup 64 sec latency 18.661 ms 2024-06-11T13:43:50.681 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:43:50.681 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:43:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:43:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:43:51.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 57977 25.30 MB/sec warmup 65 sec latency 26.035 ms 2024-06-11T13:43:51.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:51 smithi069 ceph-mon[25054]: pgmap v268: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-06-11T13:43:52.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:51 smithi135 ceph-mon[27511]: pgmap v268: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 1.27k op/s 2024-06-11T13:43:52.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 58780 25.28 MB/sec warmup 66 sec latency 24.603 ms 2024-06-11T13:43:53.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 59512 25.16 MB/sec warmup 67 sec latency 38.656 ms 2024-06-11T13:43:53.910 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:53 smithi069 ceph-mon[25054]: pgmap v269: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 959 op/s 2024-06-11T13:43:54.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:53 smithi135 ceph-mon[27511]: pgmap v269: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 959 op/s 2024-06-11T13:43:54.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 60378 25.26 MB/sec warmup 68 sec latency 21.763 ms 2024-06-11T13:43:55.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 61112 25.16 MB/sec warmup 69 sec latency 19.976 ms 2024-06-11T13:43:55.919 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:55 smithi069 ceph-mon[25054]: pgmap v270: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 12 MiB/s wr, 1.03k op/s 2024-06-11T13:43:56.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:55 smithi135 ceph-mon[27511]: pgmap v270: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 12 MiB/s wr, 1.03k op/s 2024-06-11T13:43:56.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 61830 24.98 MB/sec warmup 70 sec latency 26.957 ms 2024-06-11T13:43:57.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 62716 24.92 MB/sec warmup 71 sec latency 26.015 ms 2024-06-11T13:43:57.936 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:57 smithi069 ceph-mon[25054]: pgmap v271: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.10k op/s 2024-06-11T13:43:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:57 smithi135 ceph-mon[27511]: pgmap v271: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.10k op/s 2024-06-11T13:43:58.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 63475 24.94 MB/sec warmup 72 sec latency 14.433 ms 2024-06-11T13:43:59.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 64315 25.01 MB/sec warmup 73 sec latency 29.994 ms 2024-06-11T13:43:59.695 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:43:59 smithi069 ceph-mon[25054]: pgmap v272: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-06-11T13:44:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:43:59 smithi135 ceph-mon[27511]: pgmap v272: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.03k op/s 2024-06-11T13:44:00.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 65022 24.80 MB/sec warmup 74 sec latency 26.192 ms 2024-06-11T13:44:01.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 65815 24.78 MB/sec warmup 75 sec latency 21.878 ms 2024-06-11T13:44:01.709 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:01 smithi069 ceph-mon[25054]: pgmap v273: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-06-11T13:44:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:01 smithi135 ceph-mon[27511]: pgmap v273: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 12 MiB/s wr, 1.15k op/s 2024-06-11T13:44:02.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 66609 24.69 MB/sec warmup 76 sec latency 26.979 ms 2024-06-11T13:44:03.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 67239 24.70 MB/sec warmup 77 sec latency 50.036 ms 2024-06-11T13:44:03.811 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:03 smithi069 ceph-mon[25054]: pgmap v274: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.6 MiB/s wr, 852 op/s 2024-06-11T13:44:04.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:03 smithi135 ceph-mon[27511]: pgmap v274: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.6 MiB/s wr, 852 op/s 2024-06-11T13:44:04.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 67934 24.65 MB/sec warmup 78 sec latency 22.105 ms 2024-06-11T13:44:05.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 68570 24.44 MB/sec warmup 79 sec latency 21.598 ms 2024-06-11T13:44:05.814 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:05 smithi069 ceph-mon[25054]: pgmap v275: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-06-11T13:44:06.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:05 smithi135 ceph-mon[27511]: pgmap v275: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-06-11T13:44:06.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 69251 24.35 MB/sec warmup 80 sec latency 27.936 ms 2024-06-11T13:44:07.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 70043 24.28 MB/sec warmup 81 sec latency 25.218 ms 2024-06-11T13:44:07.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:07 smithi069 ceph-mon[25054]: pgmap v276: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2024-06-11T13:44:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:07 smithi135 ceph-mon[27511]: pgmap v276: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 1.16k op/s 2024-06-11T13:44:08.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 70714 24.33 MB/sec warmup 82 sec latency 84.308 ms 2024-06-11T13:44:09.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 71305 24.27 MB/sec warmup 83 sec latency 30.326 ms 2024-06-11T13:44:09.872 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:09 smithi069 ceph-mon[25054]: pgmap v277: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.9 MiB/s wr, 903 op/s 2024-06-11T13:44:10.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:09 smithi135 ceph-mon[27511]: pgmap v277: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.9 MiB/s wr, 903 op/s 2024-06-11T13:44:10.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 71763 24.09 MB/sec warmup 84 sec latency 31.702 ms 2024-06-11T13:44:11.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 72324 23.91 MB/sec warmup 85 sec latency 34.654 ms 2024-06-11T13:44:11.892 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:11 smithi069 ceph-mon[25054]: pgmap v278: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.14k op/s 2024-06-11T13:44:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:11 smithi135 ceph-mon[27511]: pgmap v278: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 1.14k op/s 2024-06-11T13:44:12.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 73101 23.87 MB/sec warmup 86 sec latency 37.087 ms 2024-06-11T13:44:13.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 73848 23.83 MB/sec warmup 87 sec latency 24.660 ms 2024-06-11T13:44:13.905 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:13 smithi069 ceph-mon[25054]: pgmap v279: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 879 op/s 2024-06-11T13:44:14.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:13 smithi135 ceph-mon[27511]: pgmap v279: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 879 op/s 2024-06-11T13:44:14.282 INFO:teuthology.orchestra.run.smithi069.stdout: 5 74657 23.88 MB/sec warmup 88 sec latency 19.742 ms 2024-06-11T13:44:15.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 75297 23.81 MB/sec warmup 89 sec latency 22.395 ms 2024-06-11T13:44:15.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:15 smithi069 ceph-mon[25054]: pgmap v280: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.7 MiB/s wr, 949 op/s 2024-06-11T13:44:16.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:15 smithi135 ceph-mon[27511]: pgmap v280: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.7 MiB/s wr, 949 op/s 2024-06-11T13:44:16.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 75979 23.66 MB/sec warmup 90 sec latency 33.768 ms 2024-06-11T13:44:17.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 76750 23.62 MB/sec warmup 91 sec latency 25.852 ms 2024-06-11T13:44:17.931 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:17 smithi069 ceph-mon[25054]: pgmap v281: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 991 op/s 2024-06-11T13:44:18.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:17 smithi135 ceph-mon[27511]: pgmap v281: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 991 op/s 2024-06-11T13:44:18.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 77462 23.58 MB/sec warmup 92 sec latency 22.094 ms 2024-06-11T13:44:19.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 78057 23.60 MB/sec warmup 93 sec latency 22.063 ms 2024-06-11T13:44:19.917 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:19 smithi069 ceph-mon[25054]: pgmap v282: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 887 op/s 2024-06-11T13:44:20.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:19 smithi135 ceph-mon[27511]: pgmap v282: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 887 op/s 2024-06-11T13:44:20.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 78584 23.50 MB/sec warmup 94 sec latency 24.819 ms 2024-06-11T13:44:21.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 79203 23.37 MB/sec warmup 95 sec latency 22.436 ms 2024-06-11T13:44:21.933 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:21 smithi069 ceph-mon[25054]: pgmap v283: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-06-11T13:44:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:21 smithi135 ceph-mon[27511]: pgmap v283: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-06-11T13:44:22.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 79915 23.28 MB/sec warmup 96 sec latency 28.598 ms 2024-06-11T13:44:23.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 80611 23.21 MB/sec warmup 97 sec latency 30.783 ms 2024-06-11T13:44:23.950 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:23 smithi069 ceph-mon[25054]: pgmap v284: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 731 op/s 2024-06-11T13:44:24.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:23 smithi135 ceph-mon[27511]: pgmap v284: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 731 op/s 2024-06-11T13:44:24.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 81106 23.16 MB/sec warmup 98 sec latency 28.479 ms 2024-06-11T13:44:25.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 81898 23.24 MB/sec warmup 99 sec latency 25.846 ms 2024-06-11T13:44:25.965 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:25 smithi069 ceph-mon[25054]: pgmap v285: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.6 MiB/s wr, 934 op/s 2024-06-11T13:44:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:25 smithi135 ceph-mon[27511]: pgmap v285: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.6 MiB/s wr, 934 op/s 2024-06-11T13:44:26.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 82598 23.12 MB/sec warmup 100 sec latency 21.663 ms 2024-06-11T13:44:27.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 83484 23.09 MB/sec warmup 101 sec latency 29.440 ms 2024-06-11T13:44:27.975 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:27 smithi069 ceph-mon[25054]: pgmap v286: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-06-11T13:44:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:27 smithi135 ceph-mon[27511]: pgmap v286: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-06-11T13:44:28.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 84291 23.05 MB/sec warmup 102 sec latency 33.269 ms 2024-06-11T13:44:29.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 84974 23.10 MB/sec warmup 103 sec latency 33.829 ms 2024-06-11T13:44:29.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:29 smithi069 ceph-mon[25054]: pgmap v287: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 874 op/s 2024-06-11T13:44:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:29 smithi135 ceph-mon[27511]: pgmap v287: 97 pgs: 97 active+clean; 321 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 874 op/s 2024-06-11T13:44:30.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 85531 23.05 MB/sec warmup 104 sec latency 32.892 ms 2024-06-11T13:44:31.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 85890 22.88 MB/sec warmup 105 sec latency 118.680 ms 2024-06-11T13:44:31.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:31 smithi069 ceph-mon[25054]: pgmap v288: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 974 op/s 2024-06-11T13:44:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:31 smithi135 ceph-mon[27511]: pgmap v288: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 974 op/s 2024-06-11T13:44:32.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 86547 22.78 MB/sec warmup 106 sec latency 26.638 ms 2024-06-11T13:44:33.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 87305 22.72 MB/sec warmup 107 sec latency 29.983 ms 2024-06-11T13:44:33.760 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:33 smithi069 ceph-mon[25054]: pgmap v289: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 840 op/s 2024-06-11T13:44:34.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:33 smithi135 ceph-mon[27511]: pgmap v289: 97 pgs: 97 active+clean; 324 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 840 op/s 2024-06-11T13:44:34.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 88038 22.74 MB/sec warmup 108 sec latency 24.167 ms 2024-06-11T13:44:35.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 88576 22.73 MB/sec warmup 109 sec latency 30.724 ms 2024-06-11T13:44:35.773 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:35 smithi069 ceph-mon[25054]: pgmap v290: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-06-11T13:44:36.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:35 smithi135 ceph-mon[27511]: pgmap v290: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-06-11T13:44:36.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 89212 22.69 MB/sec warmup 110 sec latency 29.127 ms 2024-06-11T13:44:37.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 89795 22.56 MB/sec warmup 111 sec latency 50.886 ms 2024-06-11T13:44:37.839 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:37 smithi069 ceph-mon[25054]: pgmap v291: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 946 op/s 2024-06-11T13:44:38.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:37 smithi135 ceph-mon[27511]: pgmap v291: 97 pgs: 97 active+clean; 319 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 946 op/s 2024-06-11T13:44:38.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 90463 22.48 MB/sec warmup 112 sec latency 30.191 ms 2024-06-11T13:44:39.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 91193 22.46 MB/sec warmup 113 sec latency 26.978 ms 2024-06-11T13:44:39.861 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:39 smithi069 ceph-mon[25054]: pgmap v292: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 815 op/s 2024-06-11T13:44:40.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:39 smithi135 ceph-mon[27511]: pgmap v292: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 815 op/s 2024-06-11T13:44:40.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 92069 22.55 MB/sec warmup 114 sec latency 35.419 ms 2024-06-11T13:44:41.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 92793 22.54 MB/sec warmup 115 sec latency 19.398 ms 2024-06-11T13:44:41.876 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:41 smithi069 ceph-mon[25054]: pgmap v293: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 974 op/s 2024-06-11T13:44:41.878 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:41 smithi135 ceph-mon[27511]: pgmap v293: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 974 op/s 2024-06-11T13:44:42.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 93469 22.45 MB/sec warmup 116 sec latency 27.497 ms 2024-06-11T13:44:43.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 94047 22.34 MB/sec warmup 117 sec latency 37.344 ms 2024-06-11T13:44:43.898 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:43 smithi069 ceph-mon[25054]: pgmap v294: 97 pgs: 97 active+clean; 320 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 789 op/s 2024-06-11T13:44:44.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:43 smithi135 ceph-mon[27511]: pgmap v294: 97 pgs: 97 active+clean; 320 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 789 op/s 2024-06-11T13:44:44.283 INFO:teuthology.orchestra.run.smithi069.stdout: 5 94632 22.28 MB/sec warmup 118 sec latency 24.925 ms 2024-06-11T13:44:45.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 95223 22.27 MB/sec warmup 119 sec latency 28.970 ms 2024-06-11T13:44:45.956 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:45 smithi069 ceph-mon[25054]: pgmap v295: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 912 op/s 2024-06-11T13:44:46.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:45 smithi135 ceph-mon[27511]: pgmap v295: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 9.3 MiB/s wr, 912 op/s 2024-06-11T13:44:47.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 96117 12.04 MB/sec execute 1 sec latency 36.247 ms 2024-06-11T13:44:47.977 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:47 smithi069 ceph-mon[25054]: pgmap v296: 97 pgs: 97 active+clean; 312 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 877 op/s 2024-06-11T13:44:48.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:47 smithi135 ceph-mon[27511]: pgmap v296: 97 pgs: 97 active+clean; 312 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 877 op/s 2024-06-11T13:44:48.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 96659 11.18 MB/sec execute 2 sec latency 46.321 ms 2024-06-11T13:44:49.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 97296 11.06 MB/sec execute 3 sec latency 43.830 ms 2024-06-11T13:44:49.988 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:49 smithi069 ceph-mon[25054]: pgmap v297: 97 pgs: 97 active+clean; 310 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 755 op/s 2024-06-11T13:44:50.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:49 smithi135 ceph-mon[27511]: pgmap v297: 97 pgs: 97 active+clean; 310 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 755 op/s 2024-06-11T13:44:50.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 97963 12.62 MB/sec execute 4 sec latency 64.439 ms 2024-06-11T13:44:50.842 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:44:50.842 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:44:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:44:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:44:51.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 98750 14.55 MB/sec execute 5 sec latency 18.848 ms 2024-06-11T13:44:52.007 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:51 smithi069 ceph-mon[25054]: pgmap v298: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 973 op/s 2024-06-11T13:44:52.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:51 smithi135 ceph-mon[27511]: pgmap v298: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 973 op/s 2024-06-11T13:44:52.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 99648 17.88 MB/sec execute 6 sec latency 24.932 ms 2024-06-11T13:44:53.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 100137 16.34 MB/sec execute 7 sec latency 41.541 ms 2024-06-11T13:44:54.022 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:53 smithi069 ceph-mon[25054]: pgmap v299: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.8 MiB/s wr, 716 op/s 2024-06-11T13:44:54.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:53 smithi135 ceph-mon[27511]: pgmap v299: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.8 MiB/s wr, 716 op/s 2024-06-11T13:44:54.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 100749 16.03 MB/sec execute 8 sec latency 35.958 ms 2024-06-11T13:44:55.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 101403 15.78 MB/sec execute 9 sec latency 37.004 ms 2024-06-11T13:44:56.031 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:55 smithi069 ceph-mon[25054]: pgmap v300: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.6 MiB/s wr, 1.00k op/s 2024-06-11T13:44:56.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:55 smithi135 ceph-mon[27511]: pgmap v300: 97 pgs: 97 active+clean; 320 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.6 MiB/s wr, 1.00k op/s 2024-06-11T13:44:56.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 102058 16.02 MB/sec execute 10 sec latency 24.154 ms 2024-06-11T13:44:57.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 102749 17.20 MB/sec execute 11 sec latency 19.407 ms 2024-06-11T13:44:57.794 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:57 smithi069 ceph-mon[25054]: pgmap v301: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 9.8 MiB/s wr, 880 op/s 2024-06-11T13:44:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:57 smithi135 ceph-mon[27511]: pgmap v301: 97 pgs: 97 active+clean; 318 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 9.8 MiB/s wr, 880 op/s 2024-06-11T13:44:58.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 103351 17.16 MB/sec execute 12 sec latency 34.060 ms 2024-06-11T13:44:59.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 103820 16.44 MB/sec execute 13 sec latency 45.209 ms 2024-06-11T13:44:59.803 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:44:59 smithi069 ceph-mon[25054]: pgmap v302: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.2 MiB/s wr, 745 op/s 2024-06-11T13:45:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:44:59 smithi135 ceph-mon[27511]: pgmap v302: 97 pgs: 97 active+clean; 317 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.2 MiB/s wr, 745 op/s 2024-06-11T13:45:00.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 104315 16.00 MB/sec execute 14 sec latency 41.989 ms 2024-06-11T13:45:01.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 104874 15.74 MB/sec execute 15 sec latency 20.635 ms 2024-06-11T13:45:01.842 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:01 smithi069 ceph-mon[25054]: pgmap v303: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-06-11T13:45:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:01 smithi135 ceph-mon[27511]: pgmap v303: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-06-11T13:45:02.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 105372 15.38 MB/sec execute 16 sec latency 29.155 ms 2024-06-11T13:45:03.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 105874 15.60 MB/sec execute 17 sec latency 35.619 ms 2024-06-11T13:45:03.823 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:03 smithi069 ceph-mon[25054]: pgmap v304: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 782 op/s 2024-06-11T13:45:04.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:03 smithi135 ceph-mon[27511]: pgmap v304: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 782 op/s 2024-06-11T13:45:04.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 106561 16.22 MB/sec execute 18 sec latency 28.495 ms 2024-06-11T13:45:05.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 106999 15.92 MB/sec execute 19 sec latency 73.797 ms 2024-06-11T13:45:05.909 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:05 smithi069 ceph-mon[25054]: pgmap v305: 97 pgs: 97 active+clean; 317 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 811 op/s 2024-06-11T13:45:06.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:05 smithi135 ceph-mon[27511]: pgmap v305: 97 pgs: 97 active+clean; 317 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 811 op/s 2024-06-11T13:45:06.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 107300 15.44 MB/sec execute 20 sec latency 51.123 ms 2024-06-11T13:45:07.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 107655 14.89 MB/sec execute 21 sec latency 42.695 ms 2024-06-11T13:45:07.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:07 smithi069 ceph-mon[25054]: pgmap v306: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 708 op/s 2024-06-11T13:45:08.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 108308 14.92 MB/sec execute 22 sec latency 43.013 ms 2024-06-11T13:45:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:07 smithi135 ceph-mon[27511]: pgmap v306: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 708 op/s 2024-06-11T13:45:09.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 109009 14.98 MB/sec execute 23 sec latency 29.167 ms 2024-06-11T13:45:09.972 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:09 smithi069 ceph-mon[25054]: pgmap v307: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 740 op/s 2024-06-11T13:45:10.097 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:09 smithi135 ceph-mon[27511]: pgmap v307: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 740 op/s 2024-06-11T13:45:10.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 109636 15.40 MB/sec execute 24 sec latency 41.385 ms 2024-06-11T13:45:11.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 110190 15.63 MB/sec execute 25 sec latency 37.274 ms 2024-06-11T13:45:11.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:11 smithi069 ceph-mon[25054]: pgmap v308: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.6 MiB/s wr, 856 op/s 2024-06-11T13:45:12.284 INFO:teuthology.orchestra.run.smithi069.stdout: 5 110590 15.35 MB/sec execute 26 sec latency 28.276 ms 2024-06-11T13:45:12.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:11 smithi135 ceph-mon[27511]: pgmap v308: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 8.6 MiB/s wr, 856 op/s 2024-06-11T13:45:13.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 111021 15.04 MB/sec execute 27 sec latency 54.437 ms 2024-06-11T13:45:14.017 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:13 smithi069 ceph-mon[25054]: pgmap v309: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 561 op/s 2024-06-11T13:45:14.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 111501 14.92 MB/sec execute 28 sec latency 51.288 ms 2024-06-11T13:45:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:13 smithi135 ceph-mon[27511]: pgmap v309: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 561 op/s 2024-06-11T13:45:15.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 112021 14.76 MB/sec execute 29 sec latency 33.893 ms 2024-06-11T13:45:15.957 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:15 smithi069 ceph-mon[25054]: pgmap v310: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.5 MiB/s wr, 728 op/s 2024-06-11T13:45:16.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 112547 14.61 MB/sec execute 30 sec latency 28.201 ms 2024-06-11T13:45:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:15 smithi135 ceph-mon[27511]: pgmap v310: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.5 MiB/s wr, 728 op/s 2024-06-11T13:45:17.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 112969 14.74 MB/sec execute 31 sec latency 28.714 ms 2024-06-11T13:45:17.994 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:17 smithi069 ceph-mon[25054]: pgmap v311: 97 pgs: 97 active+clean; 317 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 8.5 MiB/s wr, 870 op/s 2024-06-11T13:45:18.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 113417 14.91 MB/sec execute 32 sec latency 41.334 ms 2024-06-11T13:45:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:17 smithi135 ceph-mon[27511]: pgmap v311: 97 pgs: 97 active+clean; 317 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 8.5 MiB/s wr, 870 op/s 2024-06-11T13:45:19.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 113920 14.88 MB/sec execute 33 sec latency 32.454 ms 2024-06-11T13:45:20.020 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:19 smithi069 ceph-mon[25054]: pgmap v312: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.8 MiB/s wr, 695 op/s 2024-06-11T13:45:20.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 114498 14.71 MB/sec execute 34 sec latency 31.841 ms 2024-06-11T13:45:20.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:19 smithi135 ceph-mon[27511]: pgmap v312: 97 pgs: 97 active+clean; 316 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.8 MiB/s wr, 695 op/s 2024-06-11T13:45:21.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 115386 14.91 MB/sec execute 35 sec latency 21.205 ms 2024-06-11T13:45:21.870 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:21 smithi135 ceph-mon[27511]: pgmap v313: 97 pgs: 97 active+clean; 317 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 757 op/s 2024-06-11T13:45:22.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:21 smithi069 ceph-mon[25054]: pgmap v313: 97 pgs: 97 active+clean; 317 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 757 op/s 2024-06-11T13:45:22.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 116283 15.10 MB/sec execute 36 sec latency 27.541 ms 2024-06-11T13:45:23.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 117013 15.49 MB/sec execute 37 sec latency 31.010 ms 2024-06-11T13:45:24.050 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:23 smithi069 ceph-mon[25054]: pgmap v314: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-06-11T13:45:24.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 117485 15.46 MB/sec execute 38 sec latency 46.021 ms 2024-06-11T13:45:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:23 smithi135 ceph-mon[27511]: pgmap v314: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-06-11T13:45:25.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 117790 15.17 MB/sec execute 39 sec latency 42.456 ms 2024-06-11T13:45:25.830 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:25 smithi069 ceph-mon[25054]: pgmap v315: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 827 op/s 2024-06-11T13:45:25.831 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:25 smithi069 ceph-mon[25054]: pgmap v316: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 878 op/s 2024-06-11T13:45:25.831 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:25 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:45:25.831 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:25 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:45:25.831 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:25 smithi069 ceph-mon[25054]: Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-11T13:45:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:25 smithi135 ceph-mon[27511]: pgmap v315: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 827 op/s 2024-06-11T13:45:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:25 smithi135 ceph-mon[27511]: pgmap v316: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 878 op/s 2024-06-11T13:45:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:25 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:45:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:25 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:45:26.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:25 smithi135 ceph-mon[27511]: Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-11T13:45:26.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 118191 14.96 MB/sec execute 40 sec latency 35.915 ms 2024-06-11T13:45:27.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 118651 14.88 MB/sec execute 41 sec latency 61.208 ms 2024-06-11T13:45:27.889 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:27 smithi069 ceph-mon[25054]: pgmap v317: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 748 op/s 2024-06-11T13:45:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:27 smithi135 ceph-mon[27511]: pgmap v317: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 748 op/s 2024-06-11T13:45:28.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 119220 14.83 MB/sec execute 42 sec latency 38.477 ms 2024-06-11T13:45:29.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 119716 14.71 MB/sec execute 43 sec latency 28.433 ms 2024-06-11T13:45:29.922 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:29 smithi069 ceph-mon[25054]: pgmap v318: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-06-11T13:45:30.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:29 smithi135 ceph-mon[27511]: pgmap v318: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 820 op/s 2024-06-11T13:45:30.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 120228 14.88 MB/sec execute 44 sec latency 30.226 ms 2024-06-11T13:45:31.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 120717 14.97 MB/sec execute 45 sec latency 33.501 ms 2024-06-11T13:45:31.700 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:31 smithi069 ceph-mon[25054]: pgmap v319: 97 pgs: 97 active+clean; 313 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 773 op/s 2024-06-11T13:45:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:31 smithi135 ceph-mon[27511]: pgmap v319: 97 pgs: 97 active+clean; 313 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 773 op/s 2024-06-11T13:45:32.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 121073 14.88 MB/sec execute 46 sec latency 40.797 ms 2024-06-11T13:45:33.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 121410 14.69 MB/sec execute 47 sec latency 40.466 ms 2024-06-11T13:45:33.970 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:33 smithi069 ceph-mon[25054]: pgmap v320: 97 pgs: 97 active+clean; 313 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 754 op/s 2024-06-11T13:45:34.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 121883 14.49 MB/sec execute 48 sec latency 43.011 ms 2024-06-11T13:45:34.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:33 smithi135 ceph-mon[27511]: pgmap v320: 97 pgs: 97 active+clean; 313 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 754 op/s 2024-06-11T13:45:35.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 122458 14.50 MB/sec execute 49 sec latency 36.926 ms 2024-06-11T13:45:36.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:36 smithi069 ceph-mon[25054]: pgmap v321: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.4 MiB/s wr, 768 op/s 2024-06-11T13:45:36.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:45:36.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 123267 14.60 MB/sec execute 50 sec latency 25.710 ms 2024-06-11T13:45:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:36 smithi135 ceph-mon[27511]: pgmap v321: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.4 MiB/s wr, 768 op/s 2024-06-11T13:45:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:45:37.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 124160 15.00 MB/sec execute 51 sec latency 32.131 ms 2024-06-11T13:45:38.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:38 smithi069 ceph-mon[25054]: pgmap v322: 97 pgs: 97 active+clean; 322 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.8 MiB/s wr, 595 op/s 2024-06-11T13:45:38.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 124716 15.03 MB/sec execute 52 sec latency 37.335 ms 2024-06-11T13:45:38.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:38 smithi135 ceph-mon[27511]: pgmap v322: 97 pgs: 97 active+clean; 322 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.8 MiB/s wr, 595 op/s 2024-06-11T13:45:39.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 125257 14.90 MB/sec execute 53 sec latency 50.351 ms 2024-06-11T13:45:40.216 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:40 smithi069 ceph-mon[25054]: pgmap v323: 97 pgs: 97 active+clean; 317 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 554 op/s 2024-06-11T13:45:40.285 INFO:teuthology.orchestra.run.smithi069.stdout: 5 125746 14.83 MB/sec execute 54 sec latency 41.921 ms 2024-06-11T13:45:40.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:40 smithi135 ceph-mon[27511]: pgmap v323: 97 pgs: 97 active+clean; 317 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 554 op/s 2024-06-11T13:45:41.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 126348 14.84 MB/sec execute 55 sec latency 29.762 ms 2024-06-11T13:45:41.736 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:41 smithi069 ceph-mon[25054]: pgmap v324: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 823 op/s 2024-06-11T13:45:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:41 smithi135 ceph-mon[27511]: pgmap v324: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 823 op/s 2024-06-11T13:45:42.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 127231 15.03 MB/sec execute 56 sec latency 21.694 ms 2024-06-11T13:45:43.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 128100 15.32 MB/sec execute 57 sec latency 31.293 ms 2024-06-11T13:45:43.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:43 smithi069 ceph-mon[25054]: pgmap v325: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 665 op/s 2024-06-11T13:45:44.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 128728 15.26 MB/sec execute 58 sec latency 37.878 ms 2024-06-11T13:45:44.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:43 smithi135 ceph-mon[27511]: pgmap v325: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 665 op/s 2024-06-11T13:45:45.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 129250 15.17 MB/sec execute 59 sec latency 34.180 ms 2024-06-11T13:45:46.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:45 smithi069 ceph-mon[25054]: pgmap v326: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 994 op/s 2024-06-11T13:45:46.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 129822 15.17 MB/sec execute 60 sec latency 86.264 ms 2024-06-11T13:45:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:45 smithi135 ceph-mon[27511]: pgmap v326: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 994 op/s 2024-06-11T13:45:47.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 130219 15.07 MB/sec execute 61 sec latency 226.388 ms 2024-06-11T13:45:47.919 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:47 smithi069 ceph-mon[25054]: pgmap v327: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.3 MiB/s wr, 841 op/s 2024-06-11T13:45:48.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 131047 15.29 MB/sec execute 62 sec latency 28.109 ms 2024-06-11T13:45:48.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:47 smithi135 ceph-mon[27511]: pgmap v327: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.3 MiB/s wr, 841 op/s 2024-06-11T13:45:49.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 131664 15.38 MB/sec execute 63 sec latency 38.153 ms 2024-06-11T13:45:49.937 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:49 smithi069 ceph-mon[25054]: pgmap v328: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.0 MiB/s wr, 859 op/s 2024-06-11T13:45:50.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 131990 15.26 MB/sec execute 64 sec latency 47.372 ms 2024-06-11T13:45:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:49 smithi135 ceph-mon[27511]: pgmap v328: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.0 MiB/s wr, 859 op/s 2024-06-11T13:45:50.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:45:50.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:45:51.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 132360 15.08 MB/sec execute 65 sec latency 50.920 ms 2024-06-11T13:45:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:45:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:45:51.970 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:51 smithi069 ceph-mon[25054]: pgmap v329: 97 pgs: 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-06-11T13:45:52.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:51 smithi135 ceph-mon[27511]: pgmap v329: 97 pgs: 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-06-11T13:45:52.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 133234 15.20 MB/sec execute 66 sec latency 64.855 ms 2024-06-11T13:45:53.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 133974 15.23 MB/sec execute 67 sec latency 49.626 ms 2024-06-11T13:45:53.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:53 smithi069 ceph-mon[25054]: pgmap v330: 97 pgs: 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 666 op/s 2024-06-11T13:45:54.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 134313 15.18 MB/sec execute 68 sec latency 38.304 ms 2024-06-11T13:45:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:53 smithi135 ceph-mon[27511]: pgmap v330: 97 pgs: 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 666 op/s 2024-06-11T13:45:55.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 134746 15.24 MB/sec execute 69 sec latency 38.503 ms 2024-06-11T13:45:56.035 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:55 smithi069 ceph-mon[25054]: pgmap v331: 97 pgs: 97 active+clean; 324 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 9.8 MiB/s wr, 983 op/s 2024-06-11T13:45:56.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135067 15.12 MB/sec execute 70 sec latency 62.742 ms 2024-06-11T13:45:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:55 smithi135 ceph-mon[27511]: pgmap v331: 97 pgs: 97 active+clean; 324 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 9.8 MiB/s wr, 983 op/s 2024-06-11T13:45:57.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135373 15.05 MB/sec execute 71 sec latency 47.519 ms 2024-06-11T13:45:57.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:57 smithi069 ceph-mon[25054]: pgmap v332: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 710 op/s 2024-06-11T13:45:58.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135645 14.91 MB/sec execute 72 sec latency 46.972 ms 2024-06-11T13:45:58.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:57 smithi135 ceph-mon[27511]: pgmap v332: 97 pgs: 97 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 710 op/s 2024-06-11T13:45:59.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 135999 14.75 MB/sec execute 73 sec latency 43.632 ms 2024-06-11T13:45:59.892 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:45:59 smithi069 ceph-mon[25054]: pgmap v333: 97 pgs: 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 652 op/s 2024-06-11T13:46:00.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 136274 14.64 MB/sec execute 74 sec latency 59.080 ms 2024-06-11T13:46:00.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:45:59 smithi135 ceph-mon[27511]: pgmap v333: 97 pgs: 97 active+clean; 322 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 652 op/s 2024-06-11T13:46:01.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 136665 14.59 MB/sec execute 75 sec latency 52.860 ms 2024-06-11T13:46:01.680 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:01 smithi069 ceph-mon[25054]: pgmap v334: 97 pgs: 97 active+clean; 319 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 766 op/s 2024-06-11T13:46:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:01 smithi135 ceph-mon[27511]: pgmap v334: 97 pgs: 97 active+clean; 319 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 766 op/s 2024-06-11T13:46:02.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 137091 14.56 MB/sec execute 76 sec latency 31.178 ms 2024-06-11T13:46:03.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 137758 14.61 MB/sec execute 77 sec latency 25.432 ms 2024-06-11T13:46:03.953 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:03 smithi069 ceph-mon[25054]: pgmap v335: 97 pgs: 97 active+clean; 319 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 572 op/s 2024-06-11T13:46:04.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 138297 14.70 MB/sec execute 78 sec latency 24.648 ms 2024-06-11T13:46:04.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:03 smithi135 ceph-mon[27511]: pgmap v335: 97 pgs: 97 active+clean; 319 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 572 op/s 2024-06-11T13:46:05.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 138606 14.60 MB/sec execute 79 sec latency 52.881 ms 2024-06-11T13:46:05.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:05 smithi069 ceph-mon[25054]: pgmap v336: 97 pgs: 97 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 717 op/s 2024-06-11T13:46:06.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 138915 14.54 MB/sec execute 80 sec latency 49.357 ms 2024-06-11T13:46:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:05 smithi135 ceph-mon[27511]: pgmap v336: 97 pgs: 97 active+clean; 319 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 717 op/s 2024-06-11T13:46:07.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 139194 14.43 MB/sec execute 81 sec latency 53.451 ms 2024-06-11T13:46:08.014 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:07 smithi069 ceph-mon[25054]: pgmap v337: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.1 MiB/s wr, 541 op/s 2024-06-11T13:46:08.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 139651 14.36 MB/sec execute 82 sec latency 38.992 ms 2024-06-11T13:46:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:07 smithi135 ceph-mon[27511]: pgmap v337: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.1 MiB/s wr, 541 op/s 2024-06-11T13:46:09.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 140069 14.30 MB/sec execute 83 sec latency 51.702 ms 2024-06-11T13:46:10.050 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:09 smithi069 ceph-mon[25054]: pgmap v338: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 502 op/s 2024-06-11T13:46:10.097 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:09 smithi135 ceph-mon[27511]: pgmap v338: 97 pgs: 97 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 502 op/s 2024-06-11T13:46:10.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 140642 14.31 MB/sec execute 84 sec latency 35.414 ms 2024-06-11T13:46:11.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 141331 14.37 MB/sec execute 85 sec latency 26.255 ms 2024-06-11T13:46:11.822 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:11 smithi069 ceph-mon[25054]: pgmap v339: 97 pgs: 97 active+clean; 321 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 706 op/s 2024-06-11T13:46:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:11 smithi135 ceph-mon[27511]: pgmap v339: 97 pgs: 97 active+clean; 321 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 706 op/s 2024-06-11T13:46:12.286 INFO:teuthology.orchestra.run.smithi069.stdout: 5 141806 14.40 MB/sec execute 86 sec latency 37.489 ms 2024-06-11T13:46:13.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 142077 14.34 MB/sec execute 87 sec latency 53.757 ms 2024-06-11T13:46:13.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:13 smithi069 ceph-mon[25054]: pgmap v340: 97 pgs: 97 active+clean; 321 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-06-11T13:46:14.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 142366 14.28 MB/sec execute 88 sec latency 48.678 ms 2024-06-11T13:46:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:13 smithi135 ceph-mon[27511]: pgmap v340: 97 pgs: 97 active+clean; 321 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-06-11T13:46:15.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 142583 14.19 MB/sec execute 89 sec latency 49.436 ms 2024-06-11T13:46:15.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:15 smithi069 ceph-mon[25054]: pgmap v341: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 731 op/s 2024-06-11T13:46:16.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 143096 14.09 MB/sec execute 90 sec latency 38.529 ms 2024-06-11T13:46:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:15 smithi135 ceph-mon[27511]: pgmap v341: 97 pgs: 97 active+clean; 327 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 731 op/s 2024-06-11T13:46:17.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 143571 14.09 MB/sec execute 91 sec latency 39.062 ms 2024-06-11T13:46:17.983 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:17 smithi069 ceph-mon[25054]: pgmap v342: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-06-11T13:46:18.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 144073 14.07 MB/sec execute 92 sec latency 35.723 ms 2024-06-11T13:46:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:17 smithi135 ceph-mon[27511]: pgmap v342: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-06-11T13:46:19.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 144521 14.04 MB/sec execute 93 sec latency 43.227 ms 2024-06-11T13:46:20.007 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:19 smithi069 ceph-mon[25054]: pgmap v343: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 508 op/s 2024-06-11T13:46:20.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 144915 14.02 MB/sec execute 94 sec latency 36.848 ms 2024-06-11T13:46:20.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:19 smithi135 ceph-mon[27511]: pgmap v343: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 508 op/s 2024-06-11T13:46:21.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 145375 14.08 MB/sec execute 95 sec latency 39.823 ms 2024-06-11T13:46:21.802 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:21 smithi069 ceph-mon[25054]: pgmap v344: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 733 op/s 2024-06-11T13:46:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:21 smithi135 ceph-mon[27511]: pgmap v344: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 733 op/s 2024-06-11T13:46:22.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 145748 14.04 MB/sec execute 96 sec latency 36.365 ms 2024-06-11T13:46:23.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 146123 14.01 MB/sec execute 97 sec latency 35.993 ms 2024-06-11T13:46:24.013 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:23 smithi069 ceph-mon[25054]: pgmap v345: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 512 op/s 2024-06-11T13:46:24.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 146760 13.96 MB/sec execute 98 sec latency 34.463 ms 2024-06-11T13:46:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:23 smithi135 ceph-mon[27511]: pgmap v345: 97 pgs: 97 active+clean; 323 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 512 op/s 2024-06-11T13:46:25.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:24 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:46:25.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:24 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:46:25.024 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:24 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:46:25.150 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:24 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:46:25.150 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:24 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:46:25.151 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:24 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:46:25.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 147423 13.97 MB/sec execute 99 sec latency 36.983 ms 2024-06-11T13:46:26.028 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:25 smithi069 ceph-mon[25054]: pgmap v346: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 686 op/s 2024-06-11T13:46:26.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 148344 14.11 MB/sec execute 100 sec latency 30.411 ms 2024-06-11T13:46:26.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:25 smithi135 ceph-mon[27511]: pgmap v346: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 686 op/s 2024-06-11T13:46:27.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 149085 14.22 MB/sec execute 101 sec latency 22.266 ms 2024-06-11T13:46:28.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:27 smithi069 ceph-mon[25054]: pgmap v347: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 609 op/s 2024-06-11T13:46:28.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 149713 14.26 MB/sec execute 102 sec latency 31.181 ms 2024-06-11T13:46:28.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:27 smithi135 ceph-mon[27511]: pgmap v347: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 609 op/s 2024-06-11T13:46:29.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 150433 14.22 MB/sec execute 103 sec latency 26.906 ms 2024-06-11T13:46:30.058 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:29 smithi069 ceph-mon[25054]: pgmap v348: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 598 op/s 2024-06-11T13:46:30.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 151225 14.30 MB/sec execute 104 sec latency 32.906 ms 2024-06-11T13:46:30.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:29 smithi135 ceph-mon[27511]: pgmap v348: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 598 op/s 2024-06-11T13:46:31.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 152012 14.40 MB/sec execute 105 sec latency 38.229 ms 2024-06-11T13:46:31.816 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:31 smithi069 ceph-mon[25054]: pgmap v349: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.4 MiB/s wr, 934 op/s 2024-06-11T13:46:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:31 smithi135 ceph-mon[27511]: pgmap v349: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.4 MiB/s wr, 934 op/s 2024-06-11T13:46:32.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 152655 14.46 MB/sec execute 106 sec latency 29.321 ms 2024-06-11T13:46:33.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 153155 14.45 MB/sec execute 107 sec latency 35.844 ms 2024-06-11T13:46:33.827 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:33 smithi069 ceph-mon[25054]: pgmap v350: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.5 MiB/s wr, 693 op/s 2024-06-11T13:46:34.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 153751 14.43 MB/sec execute 108 sec latency 76.804 ms 2024-06-11T13:46:34.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:33 smithi135 ceph-mon[27511]: pgmap v350: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.5 MiB/s wr, 693 op/s 2024-06-11T13:46:35.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 154418 14.43 MB/sec execute 109 sec latency 30.996 ms 2024-06-11T13:46:35.849 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:35 smithi069 ceph-mon[25054]: pgmap v351: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-06-11T13:46:36.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 154902 14.41 MB/sec execute 110 sec latency 67.618 ms 2024-06-11T13:46:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:35 smithi135 ceph-mon[27511]: pgmap v351: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.02k op/s 2024-06-11T13:46:37.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 155395 14.42 MB/sec execute 111 sec latency 35.486 ms 2024-06-11T13:46:37.892 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:37 smithi069 ceph-mon[25054]: pgmap v352: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 919 op/s 2024-06-11T13:46:38.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 155718 14.38 MB/sec execute 112 sec latency 36.922 ms 2024-06-11T13:46:38.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:37 smithi135 ceph-mon[27511]: pgmap v352: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 919 op/s 2024-06-11T13:46:39.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 156304 14.44 MB/sec execute 113 sec latency 27.832 ms 2024-06-11T13:46:39.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:39 smithi069 ceph-mon[25054]: pgmap v353: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 811 op/s 2024-06-11T13:46:40.097 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:39 smithi135 ceph-mon[27511]: pgmap v353: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.9 MiB/s wr, 811 op/s 2024-06-11T13:46:40.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 156836 14.44 MB/sec execute 114 sec latency 28.742 ms 2024-06-11T13:46:41.287 INFO:teuthology.orchestra.run.smithi069.stdout: 5 157351 14.38 MB/sec execute 115 sec latency 29.902 ms 2024-06-11T13:46:41.686 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:41 smithi069 ceph-mon[25054]: pgmap v354: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-06-11T13:46:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:41 smithi135 ceph-mon[27511]: pgmap v354: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-06-11T13:46:42.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 157817 14.33 MB/sec execute 116 sec latency 42.148 ms 2024-06-11T13:46:43.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 158389 14.36 MB/sec execute 117 sec latency 37.160 ms 2024-06-11T13:46:44.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:43 smithi069 ceph-mon[25054]: pgmap v355: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 710 op/s 2024-06-11T13:46:44.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 159003 14.39 MB/sec execute 118 sec latency 31.376 ms 2024-06-11T13:46:44.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:43 smithi135 ceph-mon[27511]: pgmap v355: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 710 op/s 2024-06-11T13:46:45.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 159409 14.39 MB/sec execute 119 sec latency 25.488 ms 2024-06-11T13:46:46.065 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:45 smithi069 ceph-mon[25054]: pgmap v356: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 8.7 MiB/s wr, 882 op/s 2024-06-11T13:46:46.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 159831 14.38 MB/sec execute 120 sec latency 34.069 ms 2024-06-11T13:46:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:45 smithi135 ceph-mon[27511]: pgmap v356: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 8.7 MiB/s wr, 882 op/s 2024-06-11T13:46:47.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 160193 14.36 MB/sec execute 121 sec latency 41.935 ms 2024-06-11T13:46:48.076 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:47 smithi069 ceph-mon[25054]: pgmap v357: 97 pgs: 97 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.4 MiB/s wr, 618 op/s 2024-06-11T13:46:48.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 160948 14.34 MB/sec execute 122 sec latency 33.567 ms 2024-06-11T13:46:48.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:47 smithi135 ceph-mon[27511]: pgmap v357: 97 pgs: 97 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.4 MiB/s wr, 618 op/s 2024-06-11T13:46:49.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 161630 14.37 MB/sec execute 123 sec latency 40.956 ms 2024-06-11T13:46:49.840 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:49 smithi069 ceph-mon[25054]: pgmap v358: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.3 MiB/s wr, 716 op/s 2024-06-11T13:46:50.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 162213 14.37 MB/sec execute 124 sec latency 44.814 ms 2024-06-11T13:46:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:49 smithi135 ceph-mon[27511]: pgmap v358: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.3 MiB/s wr, 716 op/s 2024-06-11T13:46:50.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:46:50.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:46:51.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 162772 14.41 MB/sec execute 125 sec latency 26.302 ms 2024-06-11T13:46:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:46:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:46:51.859 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:51 smithi069 ceph-mon[25054]: pgmap v359: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 882 op/s 2024-06-11T13:46:52.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:51 smithi135 ceph-mon[27511]: pgmap v359: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.9 MiB/s wr, 882 op/s 2024-06-11T13:46:52.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 163263 14.44 MB/sec execute 126 sec latency 29.972 ms 2024-06-11T13:46:53.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 163679 14.42 MB/sec execute 127 sec latency 28.458 ms 2024-06-11T13:46:53.872 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:53 smithi069 ceph-mon[25054]: pgmap v360: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 594 op/s 2024-06-11T13:46:54.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 164292 14.40 MB/sec execute 128 sec latency 78.070 ms 2024-06-11T13:46:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:53 smithi135 ceph-mon[27511]: pgmap v360: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 594 op/s 2024-06-11T13:46:55.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 164865 14.37 MB/sec execute 129 sec latency 45.194 ms 2024-06-11T13:46:55.890 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:55 smithi069 ceph-mon[25054]: pgmap v361: 97 pgs: 97 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 817 op/s 2024-06-11T13:46:56.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 165344 14.37 MB/sec execute 130 sec latency 40.569 ms 2024-06-11T13:46:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:55 smithi135 ceph-mon[27511]: pgmap v361: 97 pgs: 97 active+clean; 327 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 817 op/s 2024-06-11T13:46:57.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 165832 14.34 MB/sec execute 131 sec latency 29.151 ms 2024-06-11T13:46:57.907 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:57 smithi069 ceph-mon[25054]: pgmap v362: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.2 MiB/s wr, 781 op/s 2024-06-11T13:46:58.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 166433 14.41 MB/sec execute 132 sec latency 29.118 ms 2024-06-11T13:46:58.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:57 smithi135 ceph-mon[27511]: pgmap v362: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.2 MiB/s wr, 781 op/s 2024-06-11T13:46:59.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 167060 14.47 MB/sec execute 133 sec latency 31.260 ms 2024-06-11T13:46:59.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:46:59 smithi069 ceph-mon[25054]: pgmap v363: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.1 MiB/s wr, 839 op/s 2024-06-11T13:47:00.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 167621 14.44 MB/sec execute 134 sec latency 39.599 ms 2024-06-11T13:47:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:46:59 smithi135 ceph-mon[27511]: pgmap v363: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.1 MiB/s wr, 839 op/s 2024-06-11T13:47:01.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 168271 14.43 MB/sec execute 135 sec latency 31.506 ms 2024-06-11T13:47:01.689 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:01 smithi069 ceph-mon[25054]: pgmap v364: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 873 op/s 2024-06-11T13:47:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:01 smithi135 ceph-mon[27511]: pgmap v364: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 873 op/s 2024-06-11T13:47:02.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 168874 14.43 MB/sec execute 136 sec latency 41.462 ms 2024-06-11T13:47:03.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 169522 14.45 MB/sec execute 137 sec latency 30.085 ms 2024-06-11T13:47:04.038 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:03 smithi069 ceph-mon[25054]: pgmap v365: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 684 op/s 2024-06-11T13:47:04.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 169957 14.47 MB/sec execute 138 sec latency 32.102 ms 2024-06-11T13:47:04.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:03 smithi135 ceph-mon[27511]: pgmap v365: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 684 op/s 2024-06-11T13:47:05.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 170375 14.47 MB/sec execute 139 sec latency 38.014 ms 2024-06-11T13:47:06.066 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:05 smithi069 ceph-mon[25054]: pgmap v366: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 8.2 MiB/s wr, 768 op/s 2024-06-11T13:47:06.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 170717 14.44 MB/sec execute 140 sec latency 59.677 ms 2024-06-11T13:47:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:05 smithi135 ceph-mon[27511]: pgmap v366: 97 pgs: 97 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 8.2 MiB/s wr, 768 op/s 2024-06-11T13:47:07.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 170961 14.36 MB/sec execute 141 sec latency 86.289 ms 2024-06-11T13:47:07.857 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:07 smithi069 ceph-mon[25054]: pgmap v367: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.5 MiB/s wr, 760 op/s 2024-06-11T13:47:08.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 171228 14.30 MB/sec execute 142 sec latency 56.699 ms 2024-06-11T13:47:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:07 smithi135 ceph-mon[27511]: pgmap v367: 97 pgs: 97 active+clean; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.5 MiB/s wr, 760 op/s 2024-06-11T13:47:09.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 171762 14.28 MB/sec execute 143 sec latency 52.815 ms 2024-06-11T13:47:09.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:09 smithi069 ceph-mon[25054]: pgmap v368: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 648 op/s 2024-06-11T13:47:10.097 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:09 smithi135 ceph-mon[27511]: pgmap v368: 97 pgs: 97 active+clean; 329 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 648 op/s 2024-06-11T13:47:10.288 INFO:teuthology.orchestra.run.smithi069.stdout: 5 172412 14.30 MB/sec execute 144 sec latency 36.105 ms 2024-06-11T13:47:11.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 173219 14.35 MB/sec execute 145 sec latency 30.578 ms 2024-06-11T13:47:11.638 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:11 smithi069 ceph-mon[25054]: pgmap v369: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 726 op/s 2024-06-11T13:47:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:11 smithi135 ceph-mon[27511]: pgmap v369: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 7.9 MiB/s wr, 726 op/s 2024-06-11T13:47:12.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 174002 14.44 MB/sec execute 146 sec latency 19.413 ms 2024-06-11T13:47:13.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 174533 14.43 MB/sec execute 147 sec latency 33.918 ms 2024-06-11T13:47:13.899 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:13 smithi069 ceph-mon[25054]: pgmap v370: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.5 MiB/s wr, 525 op/s 2024-06-11T13:47:14.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175045 14.40 MB/sec execute 148 sec latency 37.578 ms 2024-06-11T13:47:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:13 smithi135 ceph-mon[27511]: pgmap v370: 97 pgs: 97 active+clean; 328 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.5 MiB/s wr, 525 op/s 2024-06-11T13:47:15.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175435 14.35 MB/sec execute 149 sec latency 50.559 ms 2024-06-11T13:47:15.979 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:15 smithi069 ceph-mon[25054]: pgmap v371: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 7.6 MiB/s wr, 766 op/s 2024-06-11T13:47:16.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 175724 14.29 MB/sec execute 150 sec latency 43.240 ms 2024-06-11T13:47:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:15 smithi135 ceph-mon[27511]: pgmap v371: 97 pgs: 97 active+clean; 329 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 7.6 MiB/s wr, 766 op/s 2024-06-11T13:47:17.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 176105 14.30 MB/sec execute 151 sec latency 66.955 ms 2024-06-11T13:47:18.012 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:17 smithi069 ceph-mon[25054]: pgmap v372: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.2 MiB/s wr, 829 op/s 2024-06-11T13:47:18.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 176541 14.25 MB/sec execute 152 sec latency 43.572 ms 2024-06-11T13:47:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:17 smithi135 ceph-mon[27511]: pgmap v372: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.2 MiB/s wr, 829 op/s 2024-06-11T13:47:19.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 176878 14.26 MB/sec execute 153 sec latency 36.636 ms 2024-06-11T13:47:20.042 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:19 smithi069 ceph-mon[25054]: pgmap v373: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 633 op/s 2024-06-11T13:47:20.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 177232 14.27 MB/sec execute 154 sec latency 39.368 ms 2024-06-11T13:47:20.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:19 smithi135 ceph-mon[27511]: pgmap v373: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 633 op/s 2024-06-11T13:47:21.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 177535 14.22 MB/sec execute 155 sec latency 45.689 ms 2024-06-11T13:47:21.845 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:21 smithi069 ceph-mon[25054]: pgmap v374: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 831 op/s 2024-06-11T13:47:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:21 smithi135 ceph-mon[27511]: pgmap v374: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 831 op/s 2024-06-11T13:47:22.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 177824 14.18 MB/sec execute 156 sec latency 45.875 ms 2024-06-11T13:47:23.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 178022 14.12 MB/sec execute 157 sec latency 61.040 ms 2024-06-11T13:47:23.869 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:23 smithi069 ceph-mon[25054]: pgmap v375: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 627 op/s 2024-06-11T13:47:24.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 178257 14.06 MB/sec execute 158 sec latency 68.107 ms 2024-06-11T13:47:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:23 smithi135 ceph-mon[27511]: pgmap v375: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 627 op/s 2024-06-11T13:47:25.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 178634 14.01 MB/sec execute 159 sec latency 50.754 ms 2024-06-11T13:47:25.920 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:25 smithi069 ceph-mon[25054]: pgmap v376: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 761 op/s 2024-06-11T13:47:26.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 179014 13.97 MB/sec execute 160 sec latency 37.377 ms 2024-06-11T13:47:26.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:25 smithi135 ceph-mon[27511]: pgmap v376: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 761 op/s 2024-06-11T13:47:27.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 179375 13.95 MB/sec execute 161 sec latency 43.438 ms 2024-06-11T13:47:27.982 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:27 smithi069 ceph-mon[25054]: pgmap v377: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 6.0 MiB/s wr, 549 op/s 2024-06-11T13:47:28.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 179762 13.93 MB/sec execute 162 sec latency 95.545 ms 2024-06-11T13:47:28.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:27 smithi135 ceph-mon[27511]: pgmap v377: 97 pgs: 97 active+clean; 329 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 6.0 MiB/s wr, 549 op/s 2024-06-11T13:47:29.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 180131 13.90 MB/sec execute 163 sec latency 42.211 ms 2024-06-11T13:47:30.048 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:29 smithi069 ceph-mon[25054]: pgmap v378: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.1 MiB/s wr, 523 op/s 2024-06-11T13:47:30.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 180392 13.89 MB/sec execute 164 sec latency 64.319 ms 2024-06-11T13:47:30.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:29 smithi135 ceph-mon[27511]: pgmap v378: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.1 MiB/s wr, 523 op/s 2024-06-11T13:47:31.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 180696 13.87 MB/sec execute 165 sec latency 49.250 ms 2024-06-11T13:47:31.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:31 smithi069 ceph-mon[25054]: pgmap v379: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.7 MiB/s wr, 599 op/s 2024-06-11T13:47:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:31 smithi135 ceph-mon[27511]: pgmap v379: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.7 MiB/s wr, 599 op/s 2024-06-11T13:47:32.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 181053 13.85 MB/sec execute 166 sec latency 41.130 ms 2024-06-11T13:47:33.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 181416 13.82 MB/sec execute 167 sec latency 40.183 ms 2024-06-11T13:47:34.036 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:33 smithi069 ceph-mon[25054]: pgmap v380: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 378 op/s 2024-06-11T13:47:34.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 181718 13.79 MB/sec execute 168 sec latency 47.515 ms 2024-06-11T13:47:34.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:33 smithi135 ceph-mon[27511]: pgmap v380: 97 pgs: 97 active+clean; 326 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 378 op/s 2024-06-11T13:47:35.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 182051 13.74 MB/sec execute 169 sec latency 73.911 ms 2024-06-11T13:47:36.084 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:35 smithi069 ceph-mon[25054]: pgmap v381: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 558 op/s 2024-06-11T13:47:36.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 182383 13.68 MB/sec execute 170 sec latency 82.184 ms 2024-06-11T13:47:36.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:35 smithi135 ceph-mon[27511]: pgmap v381: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 558 op/s 2024-06-11T13:47:37.289 INFO:teuthology.orchestra.run.smithi069.stdout: 5 182662 13.65 MB/sec execute 171 sec latency 54.414 ms 2024-06-11T13:47:37.877 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:37 smithi069 ceph-mon[25054]: pgmap v382: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 452 op/s 2024-06-11T13:47:38.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 183112 13.66 MB/sec execute 172 sec latency 36.668 ms 2024-06-11T13:47:38.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:37 smithi135 ceph-mon[27511]: pgmap v382: 97 pgs: 97 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 452 op/s 2024-06-11T13:47:39.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 183556 13.62 MB/sec execute 173 sec latency 40.299 ms 2024-06-11T13:47:39.924 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:39 smithi069 ceph-mon[25054]: pgmap v383: 97 pgs: 97 active+clean; 328 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 443 op/s 2024-06-11T13:47:40.097 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:39 smithi135 ceph-mon[27511]: pgmap v383: 97 pgs: 97 active+clean; 328 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 443 op/s 2024-06-11T13:47:40.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 183937 13.63 MB/sec execute 174 sec latency 37.157 ms 2024-06-11T13:47:41.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 184292 13.63 MB/sec execute 175 sec latency 34.674 ms 2024-06-11T13:47:41.702 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:41 smithi069 ceph-mon[25054]: pgmap v384: 97 pgs: 97 active+clean; 328 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 520 op/s 2024-06-11T13:47:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:41 smithi135 ceph-mon[27511]: pgmap v384: 97 pgs: 97 active+clean; 328 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.4 MiB/s wr, 520 op/s 2024-06-11T13:47:42.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 184907 13.66 MB/sec execute 176 sec latency 25.971 ms 2024-06-11T13:47:43.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 185346 13.63 MB/sec execute 177 sec latency 34.947 ms 2024-06-11T13:47:43.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:43 smithi069 ceph-mon[25054]: pgmap v385: 97 pgs: 97 active+clean; 328 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 425 op/s 2024-06-11T13:47:44.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 185903 13.60 MB/sec execute 178 sec latency 43.853 ms 2024-06-11T13:47:44.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:43 smithi135 ceph-mon[27511]: pgmap v385: 97 pgs: 97 active+clean; 328 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 425 op/s 2024-06-11T13:47:45.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 186341 13.58 MB/sec execute 179 sec latency 43.144 ms 2024-06-11T13:47:45.983 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:45 smithi069 ceph-mon[25054]: pgmap v386: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 674 op/s 2024-06-11T13:47:46.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 187027 13.62 MB/sec execute 180 sec latency 31.236 ms 2024-06-11T13:47:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:45 smithi135 ceph-mon[27511]: pgmap v386: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 674 op/s 2024-06-11T13:47:47.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 187537 13.64 MB/sec execute 181 sec latency 32.139 ms 2024-06-11T13:47:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:47 smithi069 ceph-mon[25054]: pgmap v387: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 548 op/s 2024-06-11T13:47:48.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 188116 13.67 MB/sec execute 182 sec latency 36.846 ms 2024-06-11T13:47:48.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:47 smithi135 ceph-mon[27511]: pgmap v387: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 548 op/s 2024-06-11T13:47:49.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 188544 13.67 MB/sec execute 183 sec latency 25.401 ms 2024-06-11T13:47:50.029 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:49 smithi069 ceph-mon[25054]: pgmap v388: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 547 op/s 2024-06-11T13:47:50.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 188909 13.65 MB/sec execute 184 sec latency 41.580 ms 2024-06-11T13:47:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:49 smithi135 ceph-mon[27511]: pgmap v388: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 547 op/s 2024-06-11T13:47:51.034 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:47:51.034 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:47:51.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 189331 13.59 MB/sec execute 185 sec latency 50.534 ms 2024-06-11T13:47:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:47:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:47:52.045 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:51 smithi069 ceph-mon[25054]: pgmap v389: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 793 op/s 2024-06-11T13:47:52.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:51 smithi135 ceph-mon[27511]: pgmap v389: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 793 op/s 2024-06-11T13:47:52.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 189774 13.58 MB/sec execute 186 sec latency 43.001 ms 2024-06-11T13:47:53.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 190565 13.62 MB/sec execute 187 sec latency 35.368 ms 2024-06-11T13:47:54.059 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:53 smithi069 ceph-mon[25054]: pgmap v390: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.4 MiB/s wr, 596 op/s 2024-06-11T13:47:54.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 191282 13.68 MB/sec execute 188 sec latency 28.002 ms 2024-06-11T13:47:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:53 smithi135 ceph-mon[27511]: pgmap v390: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.4 MiB/s wr, 596 op/s 2024-06-11T13:47:55.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192079 13.74 MB/sec execute 189 sec latency 31.354 ms 2024-06-11T13:47:55.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:55 smithi069 ceph-mon[25054]: pgmap v391: 97 pgs: 97 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 813 op/s 2024-06-11T13:47:56.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192335 13.70 MB/sec execute 190 sec latency 51.608 ms 2024-06-11T13:47:56.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:55 smithi135 ceph-mon[27511]: pgmap v391: 97 pgs: 97 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 813 op/s 2024-06-11T13:47:57.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192611 13.65 MB/sec execute 191 sec latency 49.458 ms 2024-06-11T13:47:58.003 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:57 smithi069 ceph-mon[25054]: pgmap v392: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.6 MiB/s wr, 630 op/s 2024-06-11T13:47:58.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 192948 13.60 MB/sec execute 192 sec latency 48.632 ms 2024-06-11T13:47:58.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:57 smithi135 ceph-mon[27511]: pgmap v392: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.6 MiB/s wr, 630 op/s 2024-06-11T13:47:59.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 193473 13.62 MB/sec execute 193 sec latency 44.452 ms 2024-06-11T13:48:00.068 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:47:59 smithi069 ceph-mon[25054]: pgmap v393: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 644 op/s 2024-06-11T13:48:00.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 194281 13.66 MB/sec execute 194 sec latency 30.747 ms 2024-06-11T13:48:00.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:47:59 smithi135 ceph-mon[27511]: pgmap v393: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 644 op/s 2024-06-11T13:48:01.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 194667 13.65 MB/sec execute 195 sec latency 30.928 ms 2024-06-11T13:48:01.853 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:01 smithi069 ceph-mon[25054]: pgmap v394: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 8.6 MiB/s wr, 891 op/s 2024-06-11T13:48:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:01 smithi135 ceph-mon[27511]: pgmap v394: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 14 KiB/s rd, 8.6 MiB/s wr, 891 op/s 2024-06-11T13:48:02.290 INFO:teuthology.orchestra.run.smithi069.stdout: 5 195051 13.65 MB/sec execute 196 sec latency 41.151 ms 2024-06-11T13:48:03.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 195368 13.62 MB/sec execute 197 sec latency 53.837 ms 2024-06-11T13:48:03.907 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:03 smithi069 ceph-mon[25054]: pgmap v395: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 625 op/s 2024-06-11T13:48:04.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 195962 13.65 MB/sec execute 198 sec latency 40.126 ms 2024-06-11T13:48:04.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:03 smithi135 ceph-mon[27511]: pgmap v395: 97 pgs: 97 active+clean; 332 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 625 op/s 2024-06-11T13:48:05.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 196618 13.61 MB/sec execute 199 sec latency 29.999 ms 2024-06-11T13:48:05.925 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:05 smithi069 ceph-mon[25054]: pgmap v396: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 813 op/s 2024-06-11T13:48:05.926 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:05 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:48:06.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 197229 13.65 MB/sec execute 200 sec latency 48.888 ms 2024-06-11T13:48:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:05 smithi135 ceph-mon[27511]: pgmap v396: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 7.7 MiB/s wr, 813 op/s 2024-06-11T13:48:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:05 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:48:07.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 197902 13.67 MB/sec execute 201 sec latency 31.433 ms 2024-06-11T13:48:07.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:07 smithi069 ceph-mon[25054]: pgmap v397: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.8 MiB/s wr, 752 op/s 2024-06-11T13:48:08.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 198348 13.66 MB/sec execute 202 sec latency 31.945 ms 2024-06-11T13:48:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:07 smithi135 ceph-mon[27511]: pgmap v397: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.8 MiB/s wr, 752 op/s 2024-06-11T13:48:09.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 198869 13.68 MB/sec execute 203 sec latency 42.081 ms 2024-06-11T13:48:09.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:09 smithi069 ceph-mon[25054]: pgmap v398: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.1 MiB/s wr, 717 op/s 2024-06-11T13:48:10.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199124 13.65 MB/sec execute 204 sec latency 52.761 ms 2024-06-11T13:48:10.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:09 smithi135 ceph-mon[27511]: pgmap v398: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.1 MiB/s wr, 717 op/s 2024-06-11T13:48:11.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199396 13.64 MB/sec execute 205 sec latency 68.170 ms 2024-06-11T13:48:11.796 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:11 smithi069 ceph-mon[25054]: pgmap v399: 97 pgs: 97 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 894 op/s 2024-06-11T13:48:12.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:11 smithi135 ceph-mon[27511]: pgmap v399: 97 pgs: 97 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 894 op/s 2024-06-11T13:48:12.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199629 13.58 MB/sec execute 206 sec latency 71.955 ms 2024-06-11T13:48:13.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 199982 13.54 MB/sec execute 207 sec latency 60.160 ms 2024-06-11T13:48:13.866 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:13 smithi069 ceph-mon[25054]: pgmap v400: 97 pgs: 97 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.6 MiB/s wr, 618 op/s 2024-06-11T13:48:14.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 200212 13.49 MB/sec execute 208 sec latency 46.190 ms 2024-06-11T13:48:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:13 smithi135 ceph-mon[27511]: pgmap v400: 97 pgs: 97 active+clean; 335 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.6 MiB/s wr, 618 op/s 2024-06-11T13:48:15.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 200527 13.49 MB/sec execute 209 sec latency 69.305 ms 2024-06-11T13:48:15.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:15 smithi069 ceph-mon[25054]: pgmap v401: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 764 op/s 2024-06-11T13:48:16.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 200859 13.46 MB/sec execute 210 sec latency 64.683 ms 2024-06-11T13:48:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:15 smithi135 ceph-mon[27511]: pgmap v401: 97 pgs: 97 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 764 op/s 2024-06-11T13:48:17.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201123 13.43 MB/sec execute 211 sec latency 115.878 ms 2024-06-11T13:48:18.124 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:17 smithi069 ceph-mon[25054]: pgmap v402: 97 pgs: 97 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-06-11T13:48:18.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201334 13.40 MB/sec execute 212 sec latency 117.875 ms 2024-06-11T13:48:18.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:17 smithi135 ceph-mon[27511]: pgmap v402: 97 pgs: 97 active+clean; 333 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-06-11T13:48:19.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201439 13.34 MB/sec execute 213 sec latency 154.917 ms 2024-06-11T13:48:19.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:19 smithi069 ceph-mon[25054]: pgmap v403: 97 pgs: 97 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 463 op/s 2024-06-11T13:48:20.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201481 13.29 MB/sec execute 214 sec latency 281.138 ms 2024-06-11T13:48:20.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:19 smithi135 ceph-mon[27511]: pgmap v403: 97 pgs: 97 active+clean; 331 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 463 op/s 2024-06-11T13:48:21.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201555 13.24 MB/sec execute 215 sec latency 286.967 ms 2024-06-11T13:48:21.746 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:21 smithi069 ceph-mon[25054]: pgmap v404: 97 pgs: 97 active+clean; 328 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-06-11T13:48:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:21 smithi135 ceph-mon[27511]: pgmap v404: 97 pgs: 97 active+clean; 328 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-06-11T13:48:22.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201595 13.18 MB/sec execute 216 sec latency 447.168 ms 2024-06-11T13:48:23.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201619 13.13 MB/sec execute 217 sec latency 733.923 ms 2024-06-11T13:48:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:23 smithi069 ceph-mon[25054]: pgmap v405: 97 pgs: 97 active+clean; 328 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 MiB/s wr, 323 op/s 2024-06-11T13:48:24.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 13.07 MB/sec execute 218 sec latency 1443.181 ms 2024-06-11T13:48:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:23 smithi135 ceph-mon[27511]: pgmap v405: 97 pgs: 97 active+clean; 328 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 MiB/s wr, 323 op/s 2024-06-11T13:48:25.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 13.01 MB/sec execute 219 sec latency 2125.556 ms 2024-06-11T13:48:26.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:25 smithi069 ceph-mon[25054]: pgmap v406: 97 pgs: 97 active+clean; 328 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.1 MiB/s wr, 330 op/s 2024-06-11T13:48:26.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.95 MB/sec execute 220 sec latency 3125.615 ms 2024-06-11T13:48:26.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:25 smithi135 ceph-mon[27511]: pgmap v406: 97 pgs: 97 active+clean; 328 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.1 MiB/s wr, 330 op/s 2024-06-11T13:48:27.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.89 MB/sec execute 221 sec latency 4125.653 ms 2024-06-11T13:48:27.996 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:27 smithi069 ceph-mon[25054]: pgmap v407: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 MiB/s wr, 203 op/s 2024-06-11T13:48:28.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.83 MB/sec execute 222 sec latency 5125.710 ms 2024-06-11T13:48:28.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:27 smithi135 ceph-mon[27511]: pgmap v407: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 MiB/s wr, 203 op/s 2024-06-11T13:48:29.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.77 MB/sec execute 223 sec latency 6125.765 ms 2024-06-11T13:48:30.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:29 smithi069 ceph-mon[25054]: pgmap v408: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.6 MiB/s wr, 172 op/s 2024-06-11T13:48:30.291 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.72 MB/sec execute 224 sec latency 7125.825 ms 2024-06-11T13:48:30.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:29 smithi135 ceph-mon[27511]: pgmap v408: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 1.6 MiB/s wr, 172 op/s 2024-06-11T13:48:31.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.66 MB/sec execute 225 sec latency 8125.853 ms 2024-06-11T13:48:31.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:31 smithi069 ceph-mon[25054]: pgmap v409: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.3 MiB/s wr, 159 op/s 2024-06-11T13:48:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:31 smithi135 ceph-mon[27511]: pgmap v409: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.3 MiB/s wr, 159 op/s 2024-06-11T13:48:32.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.60 MB/sec execute 226 sec latency 9125.920 ms 2024-06-11T13:48:33.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.55 MB/sec execute 227 sec latency 10125.946 ms 2024-06-11T13:48:34.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:33 smithi069 ceph-mon[25054]: pgmap v410: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1.2 KiB/s rd, 253 KiB/s wr, 25 op/s 2024-06-11T13:48:34.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.49 MB/sec execute 228 sec latency 11125.975 ms 2024-06-11T13:48:34.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:33 smithi135 ceph-mon[27511]: pgmap v410: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1.2 KiB/s rd, 253 KiB/s wr, 25 op/s 2024-06-11T13:48:35.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.44 MB/sec execute 229 sec latency 12126.022 ms 2024-06-11T13:48:36.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:35 smithi069 ceph-mon[25054]: pgmap v411: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1.2 KiB/s rd, 253 KiB/s wr, 25 op/s 2024-06-11T13:48:36.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.38 MB/sec execute 230 sec latency 13126.056 ms 2024-06-11T13:48:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:35 smithi135 ceph-mon[27511]: pgmap v411: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1.2 KiB/s rd, 253 KiB/s wr, 25 op/s 2024-06-11T13:48:37.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.33 MB/sec execute 231 sec latency 14126.113 ms 2024-06-11T13:48:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:37 smithi069 ceph-mon[25054]: pgmap v412: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1.2 KiB/s rd, 131 KiB/s wr, 19 op/s 2024-06-11T13:48:38.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.28 MB/sec execute 232 sec latency 15126.165 ms 2024-06-11T13:48:38.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:37 smithi135 ceph-mon[27511]: pgmap v412: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 1.2 KiB/s rd, 131 KiB/s wr, 19 op/s 2024-06-11T13:48:39.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.22 MB/sec execute 233 sec latency 16126.214 ms 2024-06-11T13:48:40.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:39 smithi069 ceph-mon[25054]: pgmap v413: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 5.5 KiB/s wr, 0 op/s 2024-06-11T13:48:40.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.17 MB/sec execute 234 sec latency 17126.240 ms 2024-06-11T13:48:40.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:39 smithi135 ceph-mon[27511]: pgmap v413: 97 pgs: 97 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 5.5 KiB/s wr, 0 op/s 2024-06-11T13:48:41.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.12 MB/sec execute 235 sec latency 18126.300 ms 2024-06-11T13:48:41.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:41 smithi069 ceph-mon[25054]: pgmap v414: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-06-11T13:48:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:41 smithi135 ceph-mon[27511]: pgmap v414: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-06-11T13:48:42.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.07 MB/sec execute 236 sec latency 19126.327 ms 2024-06-11T13:48:43.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 12.02 MB/sec execute 237 sec latency 20126.356 ms 2024-06-11T13:48:44.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:43 smithi069 ceph-mon[25054]: pgmap v415: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-06-11T13:48:44.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 11.97 MB/sec execute 238 sec latency 21126.384 ms 2024-06-11T13:48:44.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:43 smithi135 ceph-mon[27511]: pgmap v415: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-06-11T13:48:45.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 11.92 MB/sec execute 239 sec latency 22126.439 ms 2024-06-11T13:48:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:45 smithi069 ceph-mon[25054]: pgmap v416: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-06-11T13:48:46.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 11.87 MB/sec execute 240 sec latency 23126.469 ms 2024-06-11T13:48:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:45 smithi135 ceph-mon[27511]: pgmap v416: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-06-11T13:48:46.747 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:48:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:48:46.251+0000 7f595eb00700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:25.246688+0000 front 2024-06-11T13:48:25.246276+0000 (oldest deadline 2024-06-11T13:48:46.158839+0000) 2024-06-11T13:48:46.796 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 13:48:46 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T13:48:46.398+0000 7f943d0ce700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:20.286342+0000 front 2024-06-11T13:48:20.286330+0000 (oldest deadline 2024-06-11T13:48:45.571771+0000) 2024-06-11T13:48:47.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 11.82 MB/sec execute 241 sec latency 24126.519 ms 2024-06-11T13:48:47.296 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 13:48:46 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T13:48:46.953+0000 7f85cea16700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:22.142391+0000 front 2024-06-11T13:48:22.142360+0000 (oldest deadline 2024-06-11T13:48:45.981470+0000) 2024-06-11T13:48:47.743 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:48:47 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:48:47.282+0000 7f595eb00700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:25.246688+0000 front 2024-06-11T13:48:25.246276+0000 (oldest deadline 2024-06-11T13:48:46.158839+0000) 2024-06-11T13:48:47.796 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 13:48:47 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T13:48:47.362+0000 7f943d0ce700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:20.286342+0000 front 2024-06-11T13:48:20.286330+0000 (oldest deadline 2024-06-11T13:48:45.571771+0000) 2024-06-11T13:48:47.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:47 smithi069 ceph-mon[25054]: pgmap v417: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:48:48.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 11.77 MB/sec execute 242 sec latency 25126.582 ms 2024-06-11T13:48:48.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:47 smithi135 ceph-mon[27511]: pgmap v417: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:48:48.296 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 13:48:48 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[40710]: 2024-06-11T13:48:48.084+0000 7fadd365f700 -1 osd.7 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:22.142396+0000 front 2024-06-11T13:48:22.142379+0000 (oldest deadline 2024-06-11T13:48:47.388156+0000) 2024-06-11T13:48:48.297 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 13:48:47 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T13:48:47.940+0000 7f85cea16700 -1 osd.5 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:22.142391+0000 front 2024-06-11T13:48:22.142360+0000 (oldest deadline 2024-06-11T13:48:45.981470+0000) 2024-06-11T13:48:48.497 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:48:48 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:48:48.245+0000 7f595eb00700 -1 osd.3 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:25.246688+0000 front 2024-06-11T13:48:25.246276+0000 (oldest deadline 2024-06-11T13:48:46.158839+0000) 2024-06-11T13:48:48.497 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:48:48 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T13:48:48.018+0000 7fa83e77a700 -1 osd.1 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:25.246459+0000 front 2024-06-11T13:48:25.248925+0000 (oldest deadline 2024-06-11T13:48:47.565762+0000) 2024-06-11T13:48:48.797 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 13:48:48 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T13:48:48.365+0000 7f943d0ce700 -1 osd.4 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:20.286342+0000 front 2024-06-11T13:48:20.286330+0000 (oldest deadline 2024-06-11T13:48:45.571771+0000) 2024-06-11T13:48:48.871 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:48:48 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[35117]: 2024-06-11T13:48:48.538+0000 7f190deb2700 -1 osd.0 70 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T13:48:25.246305+0000 front 2024-06-11T13:48:25.246247+0000 (oldest deadline 2024-06-11T13:48:47.911667+0000) 2024-06-11T13:48:49.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:48 smithi069 ceph-mon[25054]: osd.2 reported failed by osd.5 2024-06-11T13:48:49.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:48 smithi069 ceph-mon[25054]: osd.2 reported failed by osd.7 2024-06-11T13:48:49.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:48 smithi069 ceph-mon[25054]: osd.2 failed (root=default,host=smithi069) (2 reporters from different osd after 25.000109 >= grace 20.000000) 2024-06-11T13:48:49.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201621 11.72 MB/sec execute 243 sec latency 26126.608 ms 2024-06-11T13:48:49.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:48 smithi135 ceph-mon[27511]: osd.2 reported failed by osd.5 2024-06-11T13:48:49.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:48 smithi135 ceph-mon[27511]: osd.2 reported failed by osd.7 2024-06-11T13:48:49.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:48 smithi135 ceph-mon[27511]: osd.2 failed (root=default,host=smithi069) (2 reporters from different osd after 25.000109 >= grace 20.000000) 2024-06-11T13:48:50.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:49 smithi069 ceph-mon[25054]: pgmap v418: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:48:50.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:49 smithi069 ceph-mon[25054]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T13:48:50.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:49 smithi069 ceph-mon[25054]: osdmap e71: 8 total, 7 up, 8 in 2024-06-11T13:48:50.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:49 smithi069 ceph-mon[25054]: osdmap e72: 8 total, 7 up, 8 in 2024-06-11T13:48:50.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201685 11.68 MB/sec execute 244 sec latency 26708.535 ms 2024-06-11T13:48:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:49 smithi135 ceph-mon[27511]: pgmap v418: 97 pgs: 1 active+clean+laggy, 96 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:48:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:49 smithi135 ceph-mon[27511]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T13:48:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:49 smithi135 ceph-mon[27511]: osdmap e71: 8 total, 7 up, 8 in 2024-06-11T13:48:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:49 smithi135 ceph-mon[27511]: osdmap e72: 8 total, 7 up, 8 in 2024-06-11T13:48:51.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:48:51.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:48:51.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:48:51.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201846 11.65 MB/sec execute 245 sec latency 76.528 ms 2024-06-11T13:48:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:48:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:48:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:48:52.139 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:52 smithi069 ceph-mon[25054]: pgmap v421: 97 pgs: 8 active+undersized+degraded, 12 peering, 4 active+undersized, 73 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 12 KiB/s wr, 2 op/s; 67/1530 objects degraded (4.379%) 2024-06-11T13:48:52.139 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:52 smithi069 ceph-mon[25054]: Health check failed: Reduced data availability: 1 pg inactive, 7 pgs peering (PG_AVAILABILITY) 2024-06-11T13:48:52.139 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:52 smithi069 ceph-mon[25054]: Health check failed: Degraded data redundancy: 67/1530 objects degraded (4.379%), 8 pgs degraded (PG_DEGRADED) 2024-06-11T13:48:52.292 INFO:teuthology.orchestra.run.smithi069.stdout: 5 201952 11.61 MB/sec execute 246 sec latency 116.255 ms 2024-06-11T13:48:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:52 smithi135 ceph-mon[27511]: pgmap v421: 97 pgs: 8 active+undersized+degraded, 12 peering, 4 active+undersized, 73 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 12 KiB/s wr, 2 op/s; 67/1530 objects degraded (4.379%) 2024-06-11T13:48:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:52 smithi135 ceph-mon[27511]: Health check failed: Reduced data availability: 1 pg inactive, 7 pgs peering (PG_AVAILABILITY) 2024-06-11T13:48:52.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:52 smithi135 ceph-mon[27511]: Health check failed: Degraded data redundancy: 67/1530 objects degraded (4.379%), 8 pgs degraded (PG_DEGRADED) 2024-06-11T13:48:53.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202267 11.62 MB/sec execute 247 sec latency 206.219 ms 2024-06-11T13:48:54.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:54 smithi069 ceph-mon[25054]: pgmap v422: 97 pgs: 8 active+undersized+degraded, 12 peering, 4 active+undersized, 73 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 12 KiB/s wr, 2 op/s; 67/1530 objects degraded (4.379%) 2024-06-11T13:48:54.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202571 11.60 MB/sec execute 248 sec latency 128.982 ms 2024-06-11T13:48:54.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:54 smithi135 ceph-mon[27511]: pgmap v422: 97 pgs: 8 active+undersized+degraded, 12 peering, 4 active+undersized, 73 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 12 KiB/s wr, 2 op/s; 67/1530 objects degraded (4.379%) 2024-06-11T13:48:55.143 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:55 smithi069 ceph-mon[25054]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 7 pgs peering) 2024-06-11T13:48:55.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 202857 11.59 MB/sec execute 249 sec latency 42.296 ms 2024-06-11T13:48:55.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:55 smithi135 ceph-mon[27511]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 7 pgs peering) 2024-06-11T13:48:56.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:56 smithi069 ceph-mon[25054]: pgmap v423: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 729 KiB/s wr, 97 op/s; 141/1581 objects degraded (8.918%) 2024-06-11T13:48:56.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 203278 11.58 MB/sec execute 250 sec latency 79.370 ms 2024-06-11T13:48:56.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:56 smithi135 ceph-mon[27511]: pgmap v423: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 729 KiB/s wr, 97 op/s; 141/1581 objects degraded (8.918%) 2024-06-11T13:48:57.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:57 smithi069 ceph-mon[25054]: pgmap v424: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.0 MiB/s wr, 141 op/s; 137/1611 objects degraded (8.504%) 2024-06-11T13:48:57.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 203825 11.57 MB/sec execute 251 sec latency 26.739 ms 2024-06-11T13:48:57.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:57 smithi135 ceph-mon[27511]: pgmap v424: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 330 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.0 MiB/s wr, 141 op/s; 137/1611 objects degraded (8.504%) 2024-06-11T13:48:58.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:58 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 137/1611 objects degraded (8.504%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T13:48:58.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204248 11.59 MB/sec execute 252 sec latency 335.304 ms 2024-06-11T13:48:58.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:58 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 137/1611 objects degraded (8.504%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T13:48:59.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204249 11.54 MB/sec execute 253 sec latency 1308.121 ms 2024-06-11T13:48:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:48:59 smithi069 ceph-mon[25054]: pgmap v425: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.7 MiB/s wr, 226 op/s; 137/1632 objects degraded (8.395%) 2024-06-11T13:48:59.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:48:59 smithi135 ceph-mon[27511]: pgmap v425: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.7 MiB/s wr, 226 op/s; 137/1632 objects degraded (8.395%) 2024-06-11T13:49:00.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204249 11.49 MB/sec execute 254 sec latency 2308.148 ms 2024-06-11T13:49:01.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.45 MB/sec execute 255 sec latency 2453.205 ms 2024-06-11T13:49:01.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:01 smithi069 ceph-mon[25054]: pgmap v426: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 367 op/s; 143/1641 objects degraded (8.714%) 2024-06-11T13:49:02.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:01 smithi135 ceph-mon[27511]: pgmap v426: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 367 op/s; 143/1641 objects degraded (8.714%) 2024-06-11T13:49:02.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.40 MB/sec execute 256 sec latency 3298.942 ms 2024-06-11T13:49:03.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.36 MB/sec execute 257 sec latency 4298.968 ms 2024-06-11T13:49:04.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:03 smithi069 ceph-mon[25054]: pgmap v427: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.4 MiB/s wr, 333 op/s; 143/1641 objects degraded (8.714%) 2024-06-11T13:49:04.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.32 MB/sec execute 258 sec latency 5298.995 ms 2024-06-11T13:49:04.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:03 smithi135 ceph-mon[27511]: pgmap v427: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.4 MiB/s wr, 333 op/s; 143/1641 objects degraded (8.714%) 2024-06-11T13:49:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:04 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 137/1554 objects degraded (8.816%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:05.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.27 MB/sec execute 259 sec latency 6299.020 ms 2024-06-11T13:49:05.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:04 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 137/1554 objects degraded (8.816%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:06.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.23 MB/sec execute 260 sec latency 7299.044 ms 2024-06-11T13:49:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:06 smithi135 ceph-mon[27511]: pgmap v428: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.4 MiB/s wr, 336 op/s; 137/1554 objects degraded (8.816%) 2024-06-11T13:49:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:06.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:06 smithi069 ceph-mon[25054]: pgmap v428: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.4 MiB/s wr, 336 op/s; 137/1554 objects degraded (8.816%) 2024-06-11T13:49:06.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:07.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.19 MB/sec execute 261 sec latency 8299.070 ms 2024-06-11T13:49:08.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.14 MB/sec execute 262 sec latency 9299.098 ms 2024-06-11T13:49:08.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:08 smithi135 ceph-mon[27511]: pgmap v429: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.9 MiB/s wr, 273 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:08.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:08 smithi069 ceph-mon[25054]: pgmap v429: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.9 MiB/s wr, 273 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:09.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.10 MB/sec execute 263 sec latency 10299.161 ms 2024-06-11T13:49:10.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.06 MB/sec execute 264 sec latency 11299.187 ms 2024-06-11T13:49:10.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:10 smithi135 ceph-mon[27511]: pgmap v430: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.1 MiB/s wr, 244 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:10.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:10 smithi069 ceph-mon[25054]: pgmap v430: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.1 MiB/s wr, 244 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:11.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 11.02 MB/sec execute 265 sec latency 12299.245 ms 2024-06-11T13:49:11.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:11 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 137/1524 objects degraded (8.990%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:11 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 137/1524 objects degraded (8.990%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:12.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.98 MB/sec execute 266 sec latency 13299.271 ms 2024-06-11T13:49:12.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:12 smithi135 ceph-mon[27511]: pgmap v431: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.2 MiB/s wr, 151 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:12.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:12 smithi069 ceph-mon[25054]: pgmap v431: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.2 MiB/s wr, 151 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:13.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.94 MB/sec execute 267 sec latency 14299.297 ms 2024-06-11T13:49:14.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.89 MB/sec execute 268 sec latency 15299.330 ms 2024-06-11T13:49:14.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:14 smithi135 ceph-mon[27511]: pgmap v432: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 3 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:14.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:14 smithi069 ceph-mon[25054]: pgmap v432: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 3 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:15.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.85 MB/sec execute 269 sec latency 16299.355 ms 2024-06-11T13:49:16.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.81 MB/sec execute 270 sec latency 17299.385 ms 2024-06-11T13:49:16.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:16 smithi135 ceph-mon[27511]: pgmap v433: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 3 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:16.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:16 smithi069 ceph-mon[25054]: pgmap v433: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 3 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:17.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.77 MB/sec execute 271 sec latency 18299.421 ms 2024-06-11T13:49:18.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.73 MB/sec execute 272 sec latency 19299.448 ms 2024-06-11T13:49:18.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:18 smithi135 ceph-mon[27511]: pgmap v434: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 426 B/s wr, 1 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:18.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:18 smithi069 ceph-mon[25054]: pgmap v434: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 426 B/s wr, 1 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:19.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.69 MB/sec execute 273 sec latency 20299.475 ms 2024-06-11T13:49:20.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.66 MB/sec execute 274 sec latency 21299.505 ms 2024-06-11T13:49:20.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:20 smithi135 ceph-mon[27511]: pgmap v435: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:20.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:20 smithi069 ceph-mon[25054]: pgmap v435: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:21.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.62 MB/sec execute 275 sec latency 22299.532 ms 2024-06-11T13:49:21.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:21 smithi069 ceph-mon[25054]: pgmap v436: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:22.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:21 smithi135 ceph-mon[27511]: pgmap v436: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:22.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.58 MB/sec execute 276 sec latency 23299.570 ms 2024-06-11T13:49:22.497 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:49:22 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:49:22.091+0000 7f595eb00700 -1 osd.3 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:48:57.945643+0000 front 2024-06-11T13:48:57.946192+0000 (oldest deadline 2024-06-11T13:49:21.964097+0000) 2024-06-11T13:49:22.796 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 13:49:22 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T13:49:22.526+0000 7f85cea16700 -1 osd.5 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:49:00.439957+0000 front 2024-06-11T13:49:00.440147+0000 (oldest deadline 2024-06-11T13:49:21.689454+0000) 2024-06-11T13:49:23.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.54 MB/sec execute 277 sec latency 24299.596 ms 2024-06-11T13:49:23.497 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T13:49:23.129+0000 7f595eb00700 -1 osd.3 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:48:57.945643+0000 front 2024-06-11T13:48:57.946192+0000 (oldest deadline 2024-06-11T13:49:21.964097+0000) 2024-06-11T13:49:23.796 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T13:49:23.483+0000 7f943d0ce700 -1 osd.4 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:49:00.439770+0000 front 2024-06-11T13:49:00.439914+0000 (oldest deadline 2024-06-11T13:49:23.179398+0000) 2024-06-11T13:49:23.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T13:49:23.520+0000 7f85cea16700 -1 osd.5 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:49:00.439957+0000 front 2024-06-11T13:49:00.440147+0000 (oldest deadline 2024-06-11T13:49:21.689454+0000) 2024-06-11T13:49:23.797 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[37639]: 2024-06-11T13:49:23.701+0000 7eff1e65e700 -1 osd.6 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:48:58.982713+0000 front 2024-06-11T13:48:58.982403+0000 (oldest deadline 2024-06-11T13:49:22.899307+0000) 2024-06-11T13:49:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-mon[25054]: pgmap v437: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-mon[25054]: osd.1 reported failed by osd.3 2024-06-11T13:49:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-mon[25054]: osd.1 reported failed by osd.4 2024-06-11T13:49:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-mon[25054]: osd.1 failed (root=default,host=smithi069) (2 reporters from different osd after 23.000143 >= grace 20.000000) 2024-06-11T13:49:24.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-mon[25054]: osd.1 reported failed by osd.5 2024-06-11T13:49:24.248 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 13:49:23 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[35117]: 2024-06-11T13:49:23.823+0000 7f190deb2700 -1 osd.0 72 heartbeat_check: no reply from 172.21.15.69:6814 osd.1 since back 2024-06-11T13:48:57.946122+0000 front 2024-06-11T13:48:57.945735+0000 (oldest deadline 2024-06-11T13:49:23.714640+0000) 2024-06-11T13:49:24.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204256 10.50 MB/sec execute 278 sec latency 25299.622 ms 2024-06-11T13:49:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-mon[27511]: pgmap v437: 97 pgs: 18 active+undersized+degraded, 7 active+undersized, 72 active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 137/1524 objects degraded (8.990%) 2024-06-11T13:49:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-mon[27511]: osd.1 reported failed by osd.3 2024-06-11T13:49:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-mon[27511]: osd.1 reported failed by osd.4 2024-06-11T13:49:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-mon[27511]: osd.1 failed (root=default,host=smithi069) (2 reporters from different osd after 23.000143 >= grace 20.000000) 2024-06-11T13:49:24.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:23 smithi135 ceph-mon[27511]: osd.1 reported failed by osd.5 2024-06-11T13:49:25.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:24 smithi069 ceph-mon[25054]: Health check update: 2 osds down (OSD_DOWN) 2024-06-11T13:49:25.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:24 smithi069 ceph-mon[25054]: osdmap e73: 8 total, 6 up, 8 in 2024-06-11T13:49:25.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:24 smithi069 ceph-mon[25054]: osdmap e74: 8 total, 6 up, 8 in 2024-06-11T13:49:25.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204387 10.48 MB/sec execute 279 sec latency 25902.326 ms 2024-06-11T13:49:25.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:24 smithi135 ceph-mon[27511]: Health check update: 2 osds down (OSD_DOWN) 2024-06-11T13:49:25.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:24 smithi135 ceph-mon[27511]: osdmap e73: 8 total, 6 up, 8 in 2024-06-11T13:49:25.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:24 smithi135 ceph-mon[27511]: osdmap e74: 8 total, 6 up, 8 in 2024-06-11T13:49:26.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:25 smithi069 ceph-mon[25054]: pgmap v439: 97 pgs: 14 peering, 55 active+clean, 4 active+undersized, 15 active+undersized+degraded, 6 stale+active+clean, 2 stale+active+undersized, 1 stale+active+undersized+degraded; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 12 KiB/s wr, 1 op/s; 114/1509 objects degraded (7.555%) 2024-06-11T13:49:26.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:25 smithi069 ceph-mon[25054]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-06-11T13:49:26.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:25 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 114/1509 objects degraded (7.555%), 16 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:26.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.45 MB/sec execute 280 sec latency 1396.192 ms 2024-06-11T13:49:26.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:25 smithi135 ceph-mon[27511]: pgmap v439: 97 pgs: 14 peering, 55 active+clean, 4 active+undersized, 15 active+undersized+degraded, 6 stale+active+clean, 2 stale+active+undersized, 1 stale+active+undersized+degraded; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 12 KiB/s wr, 1 op/s; 114/1509 objects degraded (7.555%) 2024-06-11T13:49:26.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:25 smithi135 ceph-mon[27511]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-06-11T13:49:26.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:25 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 114/1509 objects degraded (7.555%), 16 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:27.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.41 MB/sec execute 281 sec latency 2396.225 ms 2024-06-11T13:49:27.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:27 smithi069 ceph-mon[25054]: pgmap v441: 97 pgs: 14 peering, 45 active+clean, 4 undersized+peered, 10 active+undersized, 2 undersized+degraded+peered, 21 active+undersized+degraded, 1 stale+active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 92 KiB/s wr, 5 op/s; 179/1485 objects degraded (12.054%) 2024-06-11T13:49:28.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.38 MB/sec execute 282 sec latency 3396.252 ms 2024-06-11T13:49:28.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:27 smithi135 ceph-mon[27511]: pgmap v441: 97 pgs: 14 peering, 45 active+clean, 4 undersized+peered, 10 active+undersized, 2 undersized+degraded+peered, 21 active+undersized+degraded, 1 stale+active+clean; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 92 KiB/s wr, 5 op/s; 179/1485 objects degraded (12.054%) 2024-06-11T13:49:29.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.34 MB/sec execute 283 sec latency 4396.279 ms 2024-06-11T13:49:30.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:29 smithi069 ceph-mon[25054]: pgmap v442: 97 pgs: 14 peering, 42 active+clean, 4 undersized+peered, 10 active+undersized, 2 undersized+degraded+peered, 25 active+undersized+degraded; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 202 KiB/s wr, 10 op/s; 200/1497 objects degraded (13.360%) 2024-06-11T13:49:30.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.30 MB/sec execute 284 sec latency 5396.304 ms 2024-06-11T13:49:30.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:29 smithi135 ceph-mon[27511]: pgmap v442: 97 pgs: 14 peering, 42 active+clean, 4 undersized+peered, 10 active+undersized, 2 undersized+degraded+peered, 25 active+undersized+degraded; 329 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 202 KiB/s wr, 10 op/s; 200/1497 objects degraded (13.360%) 2024-06-11T13:49:31.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:30 smithi069 ceph-mon[25054]: Health check update: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-06-11T13:49:31.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:30 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 200/1497 objects degraded (13.360%), 27 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:31.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.27 MB/sec execute 285 sec latency 6396.331 ms 2024-06-11T13:49:31.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:30 smithi135 ceph-mon[27511]: Health check update: Reduced data availability: 4 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-06-11T13:49:31.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:30 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 200/1497 objects degraded (13.360%), 27 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:32.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:31 smithi069 ceph-mon[25054]: pgmap v443: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 895 B/s rd, 479 KiB/s wr, 48 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:32.293 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.23 MB/sec execute 286 sec latency 7396.356 ms 2024-06-11T13:49:32.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:31 smithi135 ceph-mon[27511]: pgmap v443: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 895 B/s rd, 479 KiB/s wr, 48 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:33.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.20 MB/sec execute 287 sec latency 8396.408 ms 2024-06-11T13:49:34.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:33 smithi069 ceph-mon[25054]: pgmap v444: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 801 B/s rd, 428 KiB/s wr, 43 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:34.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.16 MB/sec execute 288 sec latency 9396.433 ms 2024-06-11T13:49:34.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:33 smithi135 ceph-mon[27511]: pgmap v444: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 801 B/s rd, 428 KiB/s wr, 43 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:35.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.13 MB/sec execute 289 sec latency 10396.457 ms 2024-06-11T13:49:36.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.09 MB/sec execute 290 sec latency 11396.486 ms 2024-06-11T13:49:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:36 smithi135 ceph-mon[27511]: pgmap v445: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 716 B/s rd, 381 KiB/s wr, 39 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:36 smithi135 ceph-mon[27511]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:49:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:36 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:36.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:36 smithi069 ceph-mon[25054]: pgmap v445: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 716 B/s rd, 381 KiB/s wr, 39 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:36.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:36.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:36 smithi069 ceph-mon[25054]: Health check update: Reduced data availability: 2 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:49:36.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:36 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded (PG_DEGRADED) 2024-06-11T13:49:37.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.06 MB/sec execute 291 sec latency 12396.515 ms 2024-06-11T13:49:38.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 10.02 MB/sec execute 292 sec latency 13396.545 ms 2024-06-11T13:49:38.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:38 smithi135 ceph-mon[27511]: pgmap v446: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 599 B/s rd, 319 KiB/s wr, 33 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:38.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:38 smithi069 ceph-mon[25054]: pgmap v446: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 599 B/s rd, 319 KiB/s wr, 33 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:39.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.99 MB/sec execute 293 sec latency 14396.580 ms 2024-06-11T13:49:40.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.95 MB/sec execute 294 sec latency 15396.611 ms 2024-06-11T13:49:40.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:40 smithi135 ceph-mon[27511]: pgmap v447: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 266 KiB/s wr, 30 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:40.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:40 smithi069 ceph-mon[25054]: pgmap v447: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 266 KiB/s wr, 30 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:41.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.92 MB/sec execute 295 sec latency 16396.647 ms 2024-06-11T13:49:41.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:41 smithi069 ceph-mon[25054]: pgmap v448: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 193 KiB/s wr, 27 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:41 smithi135 ceph-mon[27511]: pgmap v448: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 193 KiB/s wr, 27 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:42.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.89 MB/sec execute 296 sec latency 17396.679 ms 2024-06-11T13:49:43.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.85 MB/sec execute 297 sec latency 18396.711 ms 2024-06-11T13:49:44.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:43 smithi069 ceph-mon[25054]: pgmap v449: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 8.6 KiB/s wr, 1 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:44.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.82 MB/sec execute 298 sec latency 19396.746 ms 2024-06-11T13:49:44.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:43 smithi135 ceph-mon[27511]: pgmap v449: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 8.6 KiB/s wr, 1 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:45.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.79 MB/sec execute 299 sec latency 20396.776 ms 2024-06-11T13:49:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: pgmap v450: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 8.8 KiB/s wr, 2 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:46.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:46.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:46.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.76 MB/sec execute 300 sec latency 21396.811 ms 2024-06-11T13:49:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: pgmap v450: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 8.8 KiB/s wr, 2 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:49:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:46.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:46.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:46 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi069 to 1936M 2024-06-11T13:49:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:46 smithi069 ceph-mon[25054]: pgmap v451: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 189 B/s rd, 284 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:46 smithi069 ceph-mon[25054]: pgmap v452: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 232 B/s rd, 349 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:46 smithi069 ceph-mon[25054]: pgmap v453: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 301 B/s rd, 452 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:46 smithi069 ceph-mon[25054]: Health check failed: 1 hosts fail cephadm check (CEPHADM_HOST_CHECK_FAILED) 2024-06-11T13:49:47.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.72 MB/sec execute 301 sec latency 22396.845 ms 2024-06-11T13:49:47.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:46 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi069 to 1936M 2024-06-11T13:49:47.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:46 smithi135 ceph-mon[27511]: pgmap v451: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 189 B/s rd, 284 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:47.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:46 smithi135 ceph-mon[27511]: pgmap v452: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 232 B/s rd, 349 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:47.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:46 smithi135 ceph-mon[27511]: pgmap v453: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 301 B/s rd, 452 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:47.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:46 smithi135 ceph-mon[27511]: Health check failed: 1 hosts fail cephadm check (CEPHADM_HOST_CHECK_FAILED) 2024-06-11T13:49:48.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.69 MB/sec execute 302 sec latency 23396.874 ms 2024-06-11T13:49:48.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:48 smithi069 ceph-mon[25054]: pgmap v454: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 452 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:49.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.66 MB/sec execute 303 sec latency 24396.904 ms 2024-06-11T13:49:49.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:48 smithi135 ceph-mon[27511]: pgmap v454: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 452 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:50.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:49 smithi069 ceph-mon[25054]: Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:49:50.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.63 MB/sec execute 304 sec latency 25396.947 ms 2024-06-11T13:49:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:49 smithi135 ceph-mon[27511]: Health check update: Reduced data availability: 3 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:49:51.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.60 MB/sec execute 305 sec latency 26396.979 ms 2024-06-11T13:49:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:51 smithi135 ceph-mon[27511]: pgmap v455: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 452 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:49:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:49:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:51 smithi069 ceph-mon[25054]: pgmap v455: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 452 B/s wr, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:49:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:49:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:49:52.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.56 MB/sec execute 306 sec latency 27397.011 ms 2024-06-11T13:49:52.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:52 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-06-11T13:49:52.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:52 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-06-11T13:49:53.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.53 MB/sec execute 307 sec latency 28397.044 ms 2024-06-11T13:49:53.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:53 smithi069 ceph-mon[25054]: pgmap v456: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 301 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:53.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:53 smithi135 ceph-mon[27511]: pgmap v456: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 301 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:54.294 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.50 MB/sec execute 308 sec latency 29397.077 ms 2024-06-11T13:49:55.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.47 MB/sec execute 309 sec latency 30397.107 ms 2024-06-11T13:49:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:55 smithi069 ceph-mon[25054]: pgmap v457: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:55.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:55 smithi135 ceph-mon[27511]: pgmap v457: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:56.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.44 MB/sec execute 310 sec latency 31397.143 ms 2024-06-11T13:49:57.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.41 MB/sec execute 311 sec latency 32397.176 ms 2024-06-11T13:49:57.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:57 smithi069 ceph-mon[25054]: pgmap v458: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:57.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:57 smithi135 ceph-mon[27511]: pgmap v458: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:58.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.38 MB/sec execute 312 sec latency 33397.207 ms 2024-06-11T13:49:59.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.35 MB/sec execute 313 sec latency 34397.237 ms 2024-06-11T13:49:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:49:59 smithi069 ceph-mon[25054]: pgmap v459: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:49:59.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:49:59 smithi135 ceph-mon[27511]: pgmap v459: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:00.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.32 MB/sec execute 314 sec latency 35397.267 ms 2024-06-11T13:50:00.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: Health detail: HEALTH_WARN 1 hosts fail cephadm check; failed to probe daemons or devices; 2 osds down; Reduced data availability: 3 pgs inactive; Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 16 pgs undersized 2024-06-11T13:50:00.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: [WRN] CEPHADM_HOST_CHECK_FAILED: 1 hosts fail cephadm check 2024-06-11T13:50:00.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: host smithi069 (172.21.15.69) failed check: ['Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}}\n/bin/podman: stderr Error: beginning transaction: database is locked\nTraceback (most recent call last):\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in \n main()\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8622, in main\n r = ctx.func(ctx)\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 6572, in command_check_host\n engine = check_container_engine(ctx)\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine\n engine.get_version(ctx)\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version\n out, _, _ = call_throws(ctx, [self.path, \'version\', \'--format\', \'{{.Client.Version}}\'])\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws\n raise RuntimeError(f\'Failed command: {" ".join(command)}: {s}\')\nRuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked'] 2024-06-11T13:50:00.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: [WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices 2024-06-11T13:50:00.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: host smithi069 `cephadm gather-facts` failed: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}} 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: /bin/podman: stderr Error: beginning transaction: database is locked 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: Traceback (most recent call last): 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: main() 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8620, in main 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: check_container_engine(ctx) 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: engine.get_version(ctx) 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws 2024-06-11T13:50:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: raise RuntimeError(f'Failed command: {" ".join(command)}: {s}') 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: RuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: [WRN] OSD_DOWN: 2 osds down 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: osd.1 (root=default,host=smithi069) is down 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: osd.2 (root=default,host=smithi069) is down 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: [WRN] PG_AVAILABILITY: Reduced data availability: 3 pgs inactive 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.4 is stuck inactive for 68s, current state undersized+peered, last acting [3] 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.12 is stuck inactive for 7m, current state undersized+peered, last acting [6] 2024-06-11T13:50:00.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.1c is stuck inactive for 7m, current state undersized+peered, last acting [3] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: [WRN] PG_DEGRADED: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 16 pgs undersized 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.1 is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,0] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.3 is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,7] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.7 is stuck undersized for 67s, current state active+undersized+degraded, last acting [6,7] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.c is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,0] 2024-06-11T13:50:00.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.d is active+undersized+degraded, acting [4,3] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.e is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,7] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.10 is undersized+degraded+peered, acting [0] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.16 is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,6] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.17 is stuck undersized for 67s, current state active+undersized+degraded, last acting [6,5] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 2.1c is stuck undersized for 67s, current state active+undersized+degraded, last acting [4,5] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.0 is undersized+degraded+peered, acting [6] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.4 is undersized+degraded+peered, acting [5] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.5 is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,3] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.9 is stuck undersized for 67s, current state active+undersized+degraded, last acting [4,7] 2024-06-11T13:50:00.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.13 is stuck undersized for 67s, current state active+undersized+degraded, last acting [7,4] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.1a is undersized+degraded+peered, acting [4] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.1e is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,6] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 3.1f is stuck undersized for 67s, current state active+undersized+degraded, last acting [0,5] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.a is active+undersized+degraded, acting [6,7] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.17 is stuck undersized for 67s, current state active+undersized, last acting [3,7] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.18 is stuck undersized for 67s, current state active+undersized, last acting [4,6] 2024-06-11T13:50:00.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:00 smithi069 ceph-mon[25054]: pg 4.19 is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,0] 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: Health detail: HEALTH_WARN 1 hosts fail cephadm check; failed to probe daemons or devices; 2 osds down; Reduced data availability: 3 pgs inactive; Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 16 pgs undersized 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: [WRN] CEPHADM_HOST_CHECK_FAILED: 1 hosts fail cephadm check 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: host smithi069 (172.21.15.69) failed check: ['Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}}\n/bin/podman: stderr Error: beginning transaction: database is locked\nTraceback (most recent call last):\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in \n main()\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8622, in main\n r = ctx.func(ctx)\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 6572, in command_check_host\n engine = check_container_engine(ctx)\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine\n engine.get_version(ctx)\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version\n out, _, _ = call_throws(ctx, [self.path, \'version\', \'--format\', \'{{.Client.Version}}\'])\n File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws\n raise RuntimeError(f\'Failed command: {" ".join(command)}: {s}\')\nRuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked'] 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: [WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: host smithi069 `cephadm gather-facts` failed: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /bin/podman version --format {{.Client.Version}} 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: /bin/podman: stderr Error: beginning transaction: database is locked 2024-06-11T13:50:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: Traceback (most recent call last): 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8634, in 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: main() 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 8620, in main 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: check_container_engine(ctx) 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 2165, in check_container_engine 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: engine.get_version(ctx) 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 195, in get_version 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: File "/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/cephadm.ea61f01e22cb1466a53bc70973048e0ee2b182669fd1a1f6635fd7298a83a6cb", line 1615, in call_throws 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: raise RuntimeError(f'Failed command: {" ".join(command)}: {s}') 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: RuntimeError: Failed command: /bin/podman version --format {{.Client.Version}}: Error: beginning transaction: database is locked 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: [WRN] OSD_DOWN: 2 osds down 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: osd.1 (root=default,host=smithi069) is down 2024-06-11T13:50:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: osd.2 (root=default,host=smithi069) is down 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: [WRN] PG_AVAILABILITY: Reduced data availability: 3 pgs inactive 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.4 is stuck inactive for 68s, current state undersized+peered, last acting [3] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.12 is stuck inactive for 7m, current state undersized+peered, last acting [6] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.1c is stuck inactive for 7m, current state undersized+peered, last acting [3] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: [WRN] PG_DEGRADED: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 16 pgs undersized 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.1 is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,0] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.3 is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,7] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.7 is stuck undersized for 67s, current state active+undersized+degraded, last acting [6,7] 2024-06-11T13:50:00.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.a is active+undersized+degraded, acting [3,7] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.c is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,0] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.d is active+undersized+degraded, acting [4,3] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.e is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,7] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.10 is undersized+degraded+peered, acting [0] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.16 is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,6] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.17 is stuck undersized for 67s, current state active+undersized+degraded, last acting [6,5] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 2.1c is stuck undersized for 67s, current state active+undersized+degraded, last acting [4,5] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.0 is undersized+degraded+peered, acting [6] 2024-06-11T13:50:00.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.4 is undersized+degraded+peered, acting [5] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.5 is stuck undersized for 67s, current state active+undersized+degraded, last acting [5,3] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.9 is stuck undersized for 67s, current state active+undersized+degraded, last acting [4,7] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.a is active+undersized+degraded, acting [6,4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.13 is stuck undersized for 67s, current state active+undersized+degraded, last acting [7,4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.1a is undersized+degraded+peered, acting [4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-06-11T13:50:00.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.1e is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,6] 2024-06-11T13:50:00.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 3.1f is stuck undersized for 67s, current state active+undersized+degraded, last acting [0,5] 2024-06-11T13:50:00.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.a is active+undersized+degraded, acting [6,7] 2024-06-11T13:50:00.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.17 is stuck undersized for 67s, current state active+undersized, last acting [3,7] 2024-06-11T13:50:00.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.18 is stuck undersized for 67s, current state active+undersized, last acting [4,6] 2024-06-11T13:50:00.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:00 smithi135 ceph-mon[27511]: pg 4.19 is stuck undersized for 67s, current state active+undersized+degraded, last acting [3,0] 2024-06-11T13:50:01.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.29 MB/sec execute 315 sec latency 36397.301 ms 2024-06-11T13:50:01.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:01 smithi069 ceph-mon[25054]: pgmap v460: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:01.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:01 smithi135 ceph-mon[27511]: pgmap v460: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:02.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.26 MB/sec execute 316 sec latency 37397.332 ms 2024-06-11T13:50:02.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:02 smithi069 ceph-mon[25054]: pgmap v461: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:02.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:02 smithi135 ceph-mon[27511]: pgmap v461: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:03.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.23 MB/sec execute 317 sec latency 38397.366 ms 2024-06-11T13:50:04.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.20 MB/sec execute 318 sec latency 39397.398 ms 2024-06-11T13:50:04.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:04 smithi069 ceph-mon[25054]: pgmap v462: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:04 smithi135 ceph-mon[27511]: pgmap v462: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:05.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.17 MB/sec execute 319 sec latency 40397.430 ms 2024-06-11T13:50:06.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.15 MB/sec execute 320 sec latency 41397.462 ms 2024-06-11T13:50:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:06 smithi069 ceph-mon[25054]: pgmap v463: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:06 smithi135 ceph-mon[27511]: pgmap v463: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:07.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.12 MB/sec execute 321 sec latency 42397.494 ms 2024-06-11T13:50:08.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.09 MB/sec execute 322 sec latency 43397.522 ms 2024-06-11T13:50:08.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:08 smithi069 ceph-mon[25054]: pgmap v464: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:08 smithi135 ceph-mon[27511]: pgmap v464: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:09.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.06 MB/sec execute 323 sec latency 44397.555 ms 2024-06-11T13:50:10.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.03 MB/sec execute 324 sec latency 45397.597 ms 2024-06-11T13:50:10.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:10 smithi069 ceph-mon[25054]: pgmap v465: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:11.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:10 smithi135 ceph-mon[27511]: pgmap v465: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:11.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 9.00 MB/sec execute 325 sec latency 46397.631 ms 2024-06-11T13:50:12.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.98 MB/sec execute 326 sec latency 47397.660 ms 2024-06-11T13:50:12.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:12 smithi069 ceph-mon[25054]: pgmap v466: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:13.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:12 smithi135 ceph-mon[27511]: pgmap v466: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:13.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.95 MB/sec execute 327 sec latency 48397.694 ms 2024-06-11T13:50:14.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.92 MB/sec execute 328 sec latency 49397.725 ms 2024-06-11T13:50:14.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:14 smithi069 ceph-mon[25054]: pgmap v467: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:15.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:14 smithi135 ceph-mon[27511]: pgmap v467: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:15.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.90 MB/sec execute 329 sec latency 50397.758 ms 2024-06-11T13:50:16.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.87 MB/sec execute 330 sec latency 51397.799 ms 2024-06-11T13:50:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:16 smithi069 ceph-mon[25054]: pgmap v468: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:16 smithi135 ceph-mon[27511]: pgmap v468: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:17.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.84 MB/sec execute 331 sec latency 52397.836 ms 2024-06-11T13:50:18.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.81 MB/sec execute 332 sec latency 53397.868 ms 2024-06-11T13:50:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:18 smithi069 ceph-mon[25054]: pgmap v469: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:19.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:18 smithi135 ceph-mon[27511]: pgmap v469: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:19.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.79 MB/sec execute 333 sec latency 54397.902 ms 2024-06-11T13:50:20.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.76 MB/sec execute 334 sec latency 55397.933 ms 2024-06-11T13:50:20.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:20 smithi069 ceph-mon[25054]: pgmap v470: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:21.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:20 smithi135 ceph-mon[27511]: pgmap v470: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:21.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.74 MB/sec execute 335 sec latency 56397.968 ms 2024-06-11T13:50:22.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.71 MB/sec execute 336 sec latency 57397.998 ms 2024-06-11T13:50:22.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:22 smithi069 ceph-mon[25054]: pgmap v471: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:23.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:22 smithi135 ceph-mon[27511]: pgmap v471: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:23.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.68 MB/sec execute 337 sec latency 58398.033 ms 2024-06-11T13:50:24.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.66 MB/sec execute 338 sec latency 59398.064 ms 2024-06-11T13:50:24.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:24 smithi069 ceph-mon[25054]: pgmap v472: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:24 smithi135 ceph-mon[27511]: pgmap v472: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:25.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.63 MB/sec execute 339 sec latency 60398.095 ms 2024-06-11T13:50:26.295 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.61 MB/sec execute 340 sec latency 61398.126 ms 2024-06-11T13:50:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:26 smithi069 ceph-mon[25054]: pgmap v473: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:26 smithi069 ceph-mon[25054]: Health check update: Reduced data availability: 9 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:50:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:26 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 55 pgs undersized (PG_DEGRADED) 2024-06-11T13:50:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:26 smithi135 ceph-mon[27511]: pgmap v473: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:26 smithi135 ceph-mon[27511]: Health check update: Reduced data availability: 9 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:50:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:26 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 294/1482 objects degraded (19.838%), 37 pgs degraded, 55 pgs undersized (PG_DEGRADED) 2024-06-11T13:50:27.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.58 MB/sec execute 341 sec latency 62398.158 ms 2024-06-11T13:50:28.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.56 MB/sec execute 342 sec latency 63398.188 ms 2024-06-11T13:50:28.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:28 smithi069 ceph-mon[25054]: pgmap v474: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:29.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:28 smithi135 ceph-mon[27511]: pgmap v474: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:29.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.53 MB/sec execute 343 sec latency 64398.220 ms 2024-06-11T13:50:30.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.51 MB/sec execute 344 sec latency 65398.249 ms 2024-06-11T13:50:30.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:30 smithi069 ceph-mon[25054]: pgmap v475: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:30 smithi135 ceph-mon[27511]: pgmap v475: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:31.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.48 MB/sec execute 345 sec latency 66398.282 ms 2024-06-11T13:50:32.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.46 MB/sec execute 346 sec latency 67398.313 ms 2024-06-11T13:50:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:32 smithi069 ceph-mon[25054]: pgmap v476: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:33.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:32 smithi135 ceph-mon[27511]: pgmap v476: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:33.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.43 MB/sec execute 347 sec latency 68398.347 ms 2024-06-11T13:50:34.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.41 MB/sec execute 348 sec latency 69398.389 ms 2024-06-11T13:50:34.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:34 smithi069 ceph-mon[25054]: pgmap v477: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:35.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:34 smithi135 ceph-mon[27511]: pgmap v477: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:35.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.39 MB/sec execute 349 sec latency 70398.419 ms 2024-06-11T13:50:36.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.36 MB/sec execute 350 sec latency 71398.450 ms 2024-06-11T13:50:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:36 smithi069 ceph-mon[25054]: pgmap v478: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:36 smithi135 ceph-mon[27511]: pgmap v478: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:37.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.34 MB/sec execute 351 sec latency 72398.486 ms 2024-06-11T13:50:38.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.31 MB/sec execute 352 sec latency 73398.518 ms 2024-06-11T13:50:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:38 smithi069 ceph-mon[25054]: pgmap v479: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:39.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:38 smithi135 ceph-mon[27511]: pgmap v479: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:39.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.29 MB/sec execute 353 sec latency 74398.559 ms 2024-06-11T13:50:40.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.27 MB/sec execute 354 sec latency 75398.590 ms 2024-06-11T13:50:40.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:40 smithi069 ceph-mon[25054]: pgmap v480: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:41.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:40 smithi135 ceph-mon[27511]: pgmap v480: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:41.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.24 MB/sec execute 355 sec latency 76398.623 ms 2024-06-11T13:50:42.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.22 MB/sec execute 356 sec latency 77398.653 ms 2024-06-11T13:50:42.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:42 smithi069 ceph-mon[25054]: pgmap v481: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:42 smithi135 ceph-mon[27511]: pgmap v481: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:43.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.20 MB/sec execute 357 sec latency 78398.686 ms 2024-06-11T13:50:44.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.17 MB/sec execute 358 sec latency 79398.728 ms 2024-06-11T13:50:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:44 smithi069 ceph-mon[25054]: pgmap v482: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:44 smithi135 ceph-mon[27511]: pgmap v482: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:45.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.15 MB/sec execute 359 sec latency 80398.761 ms 2024-06-11T13:50:45.916 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:50:45.916 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:50:45.916 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:45 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:50:45.995 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:50:45.995 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:50:45.995 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:45 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:50:46.296 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.13 MB/sec execute 360 sec latency 81398.846 ms 2024-06-11T13:50:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:46 smithi069 ceph-mon[25054]: pgmap v483: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:47.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:46 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:47.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:46 smithi135 ceph-mon[27511]: pgmap v483: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:47.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:46 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:47.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.11 MB/sec execute 361 sec latency 82398.945 ms 2024-06-11T13:50:48.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.08 MB/sec execute 362 sec latency 83398.990 ms 2024-06-11T13:50:48.497 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T13:50:47.430+0000 7fa83e77a700 -1 osd.1 72 get_health_metrics reporting 6 slow ops, oldest is osd_op(mds.0.4:214183 2.6 2:6572afc8:::200.000000ab:head [write 238950~4737 [fadvise_dontneed] in=4737b] snapc 0=[] ondisk+write+known_if_redirected+full_force+supports_pool_eio e72) 2024-06-11T13:50:48.997 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T13:50:47.934+0000 7fc309396700 -1 osd.2 70 get_health_metrics reporting 14 slow ops, oldest is osd_op(mds.0.4:211000 2.10 2:0d7460a3:::200.000000a8:head [write 2243202~2065 [fadvise_dontneed] in=2065b] snapc 0=[] ondisk+write+known_if_redirected+full_force+supports_pool_eio e70) 2024-06-11T13:50:48.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: pgmap v484: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:48.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: pgmap v485: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:48.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:48.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:48.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: Health check cleared: CEPHADM_HOST_CHECK_FAILED (was: 1 hosts fail cephadm check) 2024-06-11T13:50:48.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: Health check cleared: CEPHADM_REFRESH_FAILED (was: failed to probe daemons or devices) 2024-06-11T13:50:48.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:48 smithi069 ceph-mon[25054]: osd.1 marked itself dead as of e74 2024-06-11T13:50:49.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: pgmap v484: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: pgmap v485: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: Health check cleared: CEPHADM_HOST_CHECK_FAILED (was: 1 hosts fail cephadm check) 2024-06-11T13:50:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: Health check cleared: CEPHADM_REFRESH_FAILED (was: failed to probe daemons or devices) 2024-06-11T13:50:49.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:48 smithi135 ceph-mon[27511]: osd.1 marked itself dead as of e74 2024-06-11T13:50:49.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.06 MB/sec execute 363 sec latency 84399.086 ms 2024-06-11T13:50:49.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:49 smithi069 ceph-mon[25054]: osdmap e75: 8 total, 6 up, 8 in 2024-06-11T13:50:49.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:49 smithi069 ceph-mon[25054]: osd.2 marked itself dead as of e75 2024-06-11T13:50:49.998 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:50:49 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T13:50:49.698+0000 7fa83cf77700 -1 osd.1 75 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:50:49.998 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:50:49 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T13:50:49.598+0000 7fc309396700 -1 osd.2 75 get_health_metrics reporting 1 slow ops, oldest is osd_op(mds.0.4:211012 2.10 2:0d7460a3:::200.000000a8:head [write 2254846~6729 [fadvise_dontneed] in=6729b] snapc 0=[] ondisk+write+known_if_redirected+full_force+supports_pool_eio e70) 2024-06-11T13:50:50.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:49 smithi135 ceph-mon[27511]: osdmap e75: 8 total, 6 up, 8 in 2024-06-11T13:50:50.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:49 smithi135 ceph-mon[27511]: osd.2 marked itself dead as of e75 2024-06-11T13:50:50.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.04 MB/sec execute 364 sec latency 85399.134 ms 2024-06-11T13:50:50.497 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T13:50:49.960+0000 7fc305b8f700 -1 osd.2 76 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T13:50:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: pgmap v487: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: Health check update: 1 osds down (OSD_DOWN) 2024-06-11T13:50:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180] boot 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: osdmap e76: 8 total, 7 up, 8 in 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448] boot 2024-06-11T13:50:51.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: osdmap e77: 8 total, 8 up, 8 in 2024-06-11T13:50:51.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:50:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: pgmap v487: 97 pgs: 42 active+clean, 5 undersized+peered, 13 active+undersized, 4 undersized+degraded+peered, 33 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 294/1482 objects degraded (19.838%) 2024-06-11T13:50:51.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: Health check update: 1 osds down (OSD_DOWN) 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: osd.1 [v2:172.21.15.69:6810/2008173180,v1:172.21.15.69:6811/2008173180] boot 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: osdmap e76: 8 total, 7 up, 8 in 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: osd.2 [v2:172.21.15.69:6818/2435076448,v1:172.21.15.69:6819/2435076448] boot 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: osdmap e77: 8 total, 8 up, 8 in 2024-06-11T13:50:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:50:51.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204485 8.02 MB/sec execute 365 sec latency 86399.165 ms 2024-06-11T13:50:52.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 204799 8.01 MB/sec execute 366 sec latency 87004.604 ms 2024-06-11T13:50:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: Monitor daemon marked osd.1 down, but it is still running 2024-06-11T13:50:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: map e74 wrongly marked me down at e73 2024-06-11T13:50:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: Monitor daemon marked osd.2 down, but it is still running 2024-06-11T13:50:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: map e75 wrongly marked me down at e71 2024-06-11T13:50:52.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: 1 slow requests (by type [ 'delayed' : 1 ] most affected pool [ 'cephfs.foofs.meta' : 1 ]) 2024-06-11T13:50:52.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: osdmap e78: 8 total, 8 up, 8 in 2024-06-11T13:50:52.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:52 smithi069 ceph-mon[25054]: pgmap v491: 97 pgs: 13 peering, 42 active+clean, 3 undersized+peered, 6 active+undersized, 3 undersized+degraded+peered, 30 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 244/1482 objects degraded (16.464%) 2024-06-11T13:50:53.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: Monitor daemon marked osd.1 down, but it is still running 2024-06-11T13:50:53.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: map e74 wrongly marked me down at e73 2024-06-11T13:50:53.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: Monitor daemon marked osd.2 down, but it is still running 2024-06-11T13:50:53.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: map e75 wrongly marked me down at e71 2024-06-11T13:50:53.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: 1 slow requests (by type [ 'delayed' : 1 ] most affected pool [ 'cephfs.foofs.meta' : 1 ]) 2024-06-11T13:50:53.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: osdmap e78: 8 total, 8 up, 8 in 2024-06-11T13:50:53.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:52 smithi135 ceph-mon[27511]: pgmap v491: 97 pgs: 13 peering, 42 active+clean, 3 undersized+peered, 6 active+undersized, 3 undersized+degraded+peered, 30 active+undersized+degraded; 331 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 244/1482 objects degraded (16.464%) 2024-06-11T13:50:53.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 205036 8.01 MB/sec execute 367 sec latency 649.192 ms 2024-06-11T13:50:53.785 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:53 smithi069 ceph-mon[25054]: Health check update: Degraded data redundancy: 244/1482 objects degraded (16.464%), 33 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-06-11T13:50:53.786 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:53 smithi069 ceph-mon[25054]: osdmap e79: 8 total, 8 up, 8 in 2024-06-11T13:50:54.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:53 smithi135 ceph-mon[27511]: Health check update: Degraded data redundancy: 244/1482 objects degraded (16.464%), 33 pgs degraded, 42 pgs undersized (PG_DEGRADED) 2024-06-11T13:50:54.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:53 smithi135 ceph-mon[27511]: osdmap e79: 8 total, 8 up, 8 in 2024-06-11T13:50:54.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 205239 8.00 MB/sec execute 368 sec latency 1024.078 ms 2024-06-11T13:50:54.795 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:54 smithi069 ceph-mon[25054]: pgmap v493: 97 pgs: 4 active+recovery_wait+degraded, 14 peering, 59 active+clean, 3 active+undersized, 1 undersized+degraded+peered, 16 active+undersized+degraded; 332 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 861 KiB/s wr, 127 op/s; 138/1506 objects degraded (9.163%); 1.1 MiB/s, 24 keys/s, 2 objects/s recovering 2024-06-11T13:50:54.795 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:54 smithi069 ceph-mon[25054]: osdmap e80: 8 total, 8 up, 8 in 2024-06-11T13:50:55.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:54 smithi135 ceph-mon[27511]: pgmap v493: 97 pgs: 4 active+recovery_wait+degraded, 14 peering, 59 active+clean, 3 active+undersized, 1 undersized+degraded+peered, 16 active+undersized+degraded; 332 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 861 KiB/s wr, 127 op/s; 138/1506 objects degraded (9.163%); 1.1 MiB/s, 24 keys/s, 2 objects/s recovering 2024-06-11T13:50:55.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:54 smithi135 ceph-mon[27511]: osdmap e80: 8 total, 8 up, 8 in 2024-06-11T13:50:55.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 205729 8.02 MB/sec execute 369 sec latency 32.838 ms 2024-06-11T13:50:55.809 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:55 smithi069 ceph-mon[25054]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:50:55.809 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:55 smithi069 ceph-mon[25054]: osdmap e81: 8 total, 8 up, 8 in 2024-06-11T13:50:56.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:55 smithi135 ceph-mon[27511]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2024-06-11T13:50:56.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:55 smithi135 ceph-mon[27511]: osdmap e81: 8 total, 8 up, 8 in 2024-06-11T13:50:56.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 206207 8.05 MB/sec execute 370 sec latency 32.903 ms 2024-06-11T13:50:56.823 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:56 smithi069 ceph-mon[25054]: pgmap v496: 97 pgs: 4 active+recovery_wait+degraded, 1 peering, 91 active+clean, 1 undersized+degraded+peered; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s wr, 253 op/s; 40/1605 objects degraded (2.492%); 2.1 MiB/s, 24 keys/s, 19 objects/s recovering 2024-06-11T13:50:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:56 smithi135 ceph-mon[27511]: pgmap v496: 97 pgs: 4 active+recovery_wait+degraded, 1 peering, 91 active+clean, 1 undersized+degraded+peered; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s wr, 253 op/s; 40/1605 objects degraded (2.492%); 2.1 MiB/s, 24 keys/s, 19 objects/s recovering 2024-06-11T13:50:57.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 206553 8.06 MB/sec execute 371 sec latency 42.780 ms 2024-06-11T13:50:58.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 206936 8.04 MB/sec execute 372 sec latency 50.295 ms 2024-06-11T13:50:58.864 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:58 smithi069 ceph-mon[25054]: pgmap v497: 97 pgs: 4 active+recovery_wait+degraded, 1 peering, 91 active+clean, 1 undersized+degraded+peered; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s wr, 169 op/s; 40/1605 objects degraded (2.492%); 1.4 MiB/s, 16 keys/s, 13 objects/s recovering 2024-06-11T13:50:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:58 smithi135 ceph-mon[27511]: pgmap v497: 97 pgs: 4 active+recovery_wait+degraded, 1 peering, 91 active+clean, 1 undersized+degraded+peered; 333 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s wr, 169 op/s; 40/1605 objects degraded (2.492%); 1.4 MiB/s, 16 keys/s, 13 objects/s recovering 2024-06-11T13:50:59.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 207255 8.03 MB/sec execute 373 sec latency 90.093 ms 2024-06-11T13:50:59.916 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:59 smithi069 ceph-mon[25054]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-06-11T13:50:59.916 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:59 smithi069 ceph-mon[25054]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/1605 objects degraded (2.492%), 5 pgs degraded, 1 pg undersized) 2024-06-11T13:50:59.916 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:50:59 smithi069 ceph-mon[25054]: Cluster is now healthy 2024-06-11T13:51:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:59 smithi135 ceph-mon[27511]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2024-06-11T13:51:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:59 smithi135 ceph-mon[27511]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/1605 objects degraded (2.492%), 5 pgs degraded, 1 pg undersized) 2024-06-11T13:51:00.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:50:59 smithi135 ceph-mon[27511]: Cluster is now healthy 2024-06-11T13:51:00.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 207611 8.05 MB/sec execute 374 sec latency 125.253 ms 2024-06-11T13:51:00.951 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:00 smithi069 ceph-mon[25054]: pgmap v498: 97 pgs: 97 active+clean; 327 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.3 MiB/s wr, 312 op/s; 1.5 MiB/s, 13 keys/s, 14 objects/s recovering 2024-06-11T13:51:01.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:00 smithi135 ceph-mon[27511]: pgmap v498: 97 pgs: 97 active+clean; 327 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.3 MiB/s wr, 312 op/s; 1.5 MiB/s, 13 keys/s, 14 objects/s recovering 2024-06-11T13:51:01.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 208003 8.06 MB/sec execute 375 sec latency 50.195 ms 2024-06-11T13:51:02.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 208440 8.07 MB/sec execute 376 sec latency 43.394 ms 2024-06-11T13:51:02.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:02 smithi069 ceph-mon[25054]: pgmap v499: 97 pgs: 97 active+clean; 329 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 379 op/s; 748 KiB/s, 11 objects/s recovering 2024-06-11T13:51:03.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:02 smithi135 ceph-mon[27511]: pgmap v499: 97 pgs: 97 active+clean; 329 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 379 op/s; 748 KiB/s, 11 objects/s recovering 2024-06-11T13:51:03.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 208740 8.06 MB/sec execute 377 sec latency 38.226 ms 2024-06-11T13:51:04.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 209035 8.07 MB/sec execute 378 sec latency 43.786 ms 2024-06-11T13:51:04.767 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:04 smithi069 ceph-mon[25054]: pgmap v500: 97 pgs: 97 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 511 op/s; 599 KiB/s, 9 objects/s recovering 2024-06-11T13:51:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:04 smithi135 ceph-mon[27511]: pgmap v500: 97 pgs: 97 active+clean; 332 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 511 op/s; 599 KiB/s, 9 objects/s recovering 2024-06-11T13:51:05.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 209359 8.07 MB/sec execute 379 sec latency 48.948 ms 2024-06-11T13:51:06.057 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:51:06.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:51:06.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 209659 8.07 MB/sec execute 380 sec latency 63.165 ms 2024-06-11T13:51:07.068 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:07 smithi069 ceph-mon[25054]: pgmap v501: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 546 op/s; 544 KiB/s, 6 objects/s recovering 2024-06-11T13:51:07.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:07 smithi135 ceph-mon[27511]: pgmap v501: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 546 op/s; 544 KiB/s, 6 objects/s recovering 2024-06-11T13:51:07.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 210040 8.09 MB/sec execute 381 sec latency 51.941 ms 2024-06-11T13:51:08.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 210717 8.09 MB/sec execute 382 sec latency 19.737 ms 2024-06-11T13:51:09.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:09 smithi069 ceph-mon[25054]: pgmap v502: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 462 op/s; 158 KiB/s, 2 objects/s recovering 2024-06-11T13:51:09.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:09 smithi135 ceph-mon[27511]: pgmap v502: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 462 op/s; 158 KiB/s, 2 objects/s recovering 2024-06-11T13:51:09.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 211508 8.14 MB/sec execute 383 sec latency 22.514 ms 2024-06-11T13:51:10.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 212192 8.16 MB/sec execute 384 sec latency 65.028 ms 2024-06-11T13:51:11.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:11 smithi069 ceph-mon[25054]: pgmap v503: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.2 MiB/s wr, 677 op/s; 158 KiB/s, 2 objects/s recovering 2024-06-11T13:51:11.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:11 smithi135 ceph-mon[27511]: pgmap v503: 97 pgs: 97 active+clean; 344 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.2 MiB/s wr, 677 op/s; 158 KiB/s, 2 objects/s recovering 2024-06-11T13:51:11.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 212650 8.17 MB/sec execute 385 sec latency 36.141 ms 2024-06-11T13:51:12.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213163 8.19 MB/sec execute 386 sec latency 54.871 ms 2024-06-11T13:51:13.165 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:13 smithi069 ceph-mon[25054]: pgmap v504: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.4 MiB/s wr, 640 op/s 2024-06-11T13:51:13.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:13 smithi135 ceph-mon[27511]: pgmap v504: 97 pgs: 97 active+clean; 336 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.4 MiB/s wr, 640 op/s 2024-06-11T13:51:13.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 213644 8.22 MB/sec execute 387 sec latency 47.775 ms 2024-06-11T13:51:14.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 214116 8.20 MB/sec execute 388 sec latency 55.953 ms 2024-06-11T13:51:15.125 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:15 smithi069 ceph-mon[25054]: pgmap v505: 97 pgs: 97 active+clean; 336 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-06-11T13:51:15.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:15 smithi135 ceph-mon[27511]: pgmap v505: 97 pgs: 97 active+clean; 336 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-06-11T13:51:15.297 INFO:teuthology.orchestra.run.smithi069.stdout: 5 214497 8.21 MB/sec execute 389 sec latency 62.710 ms 2024-06-11T13:51:16.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 214976 8.23 MB/sec execute 390 sec latency 46.084 ms 2024-06-11T13:51:16.667 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:16 smithi069 ceph-mon[25054]: pgmap v506: 97 pgs: 97 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.3 MiB/s wr, 670 op/s 2024-06-11T13:51:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:16 smithi135 ceph-mon[27511]: pgmap v506: 97 pgs: 97 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.3 MiB/s wr, 670 op/s 2024-06-11T13:51:17.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 215514 8.24 MB/sec execute 391 sec latency 41.541 ms 2024-06-11T13:51:18.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 215920 8.24 MB/sec execute 392 sec latency 70.827 ms 2024-06-11T13:51:18.971 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:18 smithi069 ceph-mon[25054]: pgmap v507: 97 pgs: 97 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 591 op/s 2024-06-11T13:51:19.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:18 smithi135 ceph-mon[27511]: pgmap v507: 97 pgs: 97 active+clean; 337 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 591 op/s 2024-06-11T13:51:19.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 216432 8.26 MB/sec execute 393 sec latency 33.577 ms 2024-06-11T13:51:20.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 216828 8.28 MB/sec execute 394 sec latency 35.214 ms 2024-06-11T13:51:20.766 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:20 smithi069 ceph-mon[25054]: pgmap v508: 97 pgs: 97 active+clean; 336 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 822 op/s 2024-06-11T13:51:21.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:20 smithi135 ceph-mon[27511]: pgmap v508: 97 pgs: 97 active+clean; 336 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 822 op/s 2024-06-11T13:51:21.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217072 8.28 MB/sec execute 395 sec latency 93.405 ms 2024-06-11T13:51:22.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217319 8.27 MB/sec execute 396 sec latency 55.193 ms 2024-06-11T13:51:22.820 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:22 smithi069 ceph-mon[25054]: pgmap v509: 97 pgs: 97 active+clean; 336 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.3 MiB/s wr, 654 op/s 2024-06-11T13:51:23.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:22 smithi135 ceph-mon[27511]: pgmap v509: 97 pgs: 97 active+clean; 336 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.3 MiB/s wr, 654 op/s 2024-06-11T13:51:23.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217558 8.26 MB/sec execute 397 sec latency 82.694 ms 2024-06-11T13:51:24.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217753 8.24 MB/sec execute 398 sec latency 124.241 ms 2024-06-11T13:51:24.876 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:24 smithi069 ceph-mon[25054]: pgmap v510: 97 pgs: 97 active+clean; 333 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 620 op/s 2024-06-11T13:51:25.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:24 smithi135 ceph-mon[27511]: pgmap v510: 97 pgs: 97 active+clean; 333 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 620 op/s 2024-06-11T13:51:25.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217848 8.23 MB/sec execute 399 sec latency 136.799 ms 2024-06-11T13:51:26.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 217939 8.21 MB/sec execute 400 sec latency 208.124 ms 2024-06-11T13:51:26.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:26 smithi069 ceph-mon[25054]: pgmap v511: 97 pgs: 97 active+clean; 337 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.8 MiB/s wr, 718 op/s 2024-06-11T13:51:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:26 smithi135 ceph-mon[27511]: pgmap v511: 97 pgs: 97 active+clean; 337 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.8 MiB/s wr, 718 op/s 2024-06-11T13:51:27.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218027 8.20 MB/sec execute 401 sec latency 184.278 ms 2024-06-11T13:51:28.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218060 8.18 MB/sec execute 402 sec latency 313.964 ms 2024-06-11T13:51:28.849 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:28 smithi069 ceph-mon[25054]: pgmap v512: 97 pgs: 97 active+clean; 337 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-06-11T13:51:29.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:28 smithi135 ceph-mon[27511]: pgmap v512: 97 pgs: 97 active+clean; 337 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-06-11T13:51:29.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218159 8.17 MB/sec execute 403 sec latency 129.462 ms 2024-06-11T13:51:30.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218310 8.16 MB/sec execute 404 sec latency 129.026 ms 2024-06-11T13:51:30.975 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:30 smithi069 ceph-mon[25054]: pgmap v513: 97 pgs: 97 active+clean; 333 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.9 MiB/s wr, 502 op/s 2024-06-11T13:51:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:30 smithi135 ceph-mon[27511]: pgmap v513: 97 pgs: 97 active+clean; 333 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.9 MiB/s wr, 502 op/s 2024-06-11T13:51:31.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218404 8.15 MB/sec execute 405 sec latency 134.301 ms 2024-06-11T13:51:32.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218511 8.14 MB/sec execute 406 sec latency 188.417 ms 2024-06-11T13:51:32.776 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:32 smithi069 ceph-mon[25054]: pgmap v514: 97 pgs: 97 active+clean; 330 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 287 op/s 2024-06-11T13:51:33.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:32 smithi135 ceph-mon[27511]: pgmap v514: 97 pgs: 97 active+clean; 330 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.5 MiB/s wr, 287 op/s 2024-06-11T13:51:33.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218644 8.13 MB/sec execute 407 sec latency 193.673 ms 2024-06-11T13:51:34.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 218761 8.12 MB/sec execute 408 sec latency 166.106 ms 2024-06-11T13:51:34.966 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:34 smithi069 ceph-mon[25054]: pgmap v515: 97 pgs: 97 active+clean; 332 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.7 MiB/s wr, 291 op/s 2024-06-11T13:51:35.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:34 smithi135 ceph-mon[27511]: pgmap v515: 97 pgs: 97 active+clean; 332 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.7 MiB/s wr, 291 op/s 2024-06-11T13:51:35.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 219254 8.12 MB/sec execute 409 sec latency 100.054 ms 2024-06-11T13:51:36.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 219587 8.12 MB/sec execute 410 sec latency 27.943 ms 2024-06-11T13:51:36.762 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:36 smithi069 ceph-mon[25054]: pgmap v516: 97 pgs: 97 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.6 MiB/s wr, 311 op/s 2024-06-11T13:51:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:36 smithi135 ceph-mon[27511]: pgmap v516: 97 pgs: 97 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.6 MiB/s wr, 311 op/s 2024-06-11T13:51:37.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 220063 8.14 MB/sec execute 411 sec latency 37.286 ms 2024-06-11T13:51:38.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 220407 8.15 MB/sec execute 412 sec latency 58.441 ms 2024-06-11T13:51:38.804 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:38 smithi069 ceph-mon[25054]: pgmap v517: 97 pgs: 97 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.5 MiB/s wr, 168 op/s 2024-06-11T13:51:39.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:38 smithi135 ceph-mon[27511]: pgmap v517: 97 pgs: 97 active+clean; 334 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.5 MiB/s wr, 168 op/s 2024-06-11T13:51:39.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 220721 8.15 MB/sec execute 413 sec latency 57.540 ms 2024-06-11T13:51:40.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 221058 8.15 MB/sec execute 414 sec latency 41.998 ms 2024-06-11T13:51:40.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:40 smithi069 ceph-mon[25054]: pgmap v518: 97 pgs: 97 active+clean; 339 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.5 MiB/s wr, 242 op/s 2024-06-11T13:51:41.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:40 smithi135 ceph-mon[27511]: pgmap v518: 97 pgs: 97 active+clean; 339 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.5 MiB/s wr, 242 op/s 2024-06-11T13:51:41.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 221483 8.15 MB/sec execute 415 sec latency 35.282 ms 2024-06-11T13:51:42.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 221827 8.15 MB/sec execute 416 sec latency 32.844 ms 2024-06-11T13:51:42.882 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:42 smithi069 ceph-mon[25054]: pgmap v519: 97 pgs: 97 active+clean; 339 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-06-11T13:51:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:42 smithi135 ceph-mon[27511]: pgmap v519: 97 pgs: 97 active+clean; 339 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.3 MiB/s wr, 293 op/s 2024-06-11T13:51:43.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 222232 8.17 MB/sec execute 417 sec latency 41.009 ms 2024-06-11T13:51:44.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 222786 8.17 MB/sec execute 418 sec latency 69.326 ms 2024-06-11T13:51:44.908 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:44 smithi069 ceph-mon[25054]: pgmap v520: 97 pgs: 97 active+clean; 335 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 364 op/s 2024-06-11T13:51:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:44 smithi135 ceph-mon[27511]: pgmap v520: 97 pgs: 97 active+clean; 335 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 364 op/s 2024-06-11T13:51:45.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223152 8.17 MB/sec execute 419 sec latency 35.379 ms 2024-06-11T13:51:46.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223451 8.18 MB/sec execute 420 sec latency 128.890 ms 2024-06-11T13:51:46.902 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:46 smithi069 ceph-mon[25054]: pgmap v521: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-06-11T13:51:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:46 smithi135 ceph-mon[27511]: pgmap v521: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-06-11T13:51:47.298 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223597 8.17 MB/sec execute 421 sec latency 175.276 ms 2024-06-11T13:51:47.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:51:47.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:51:47.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:47 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:51:47.964 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:51:47.964 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:51:47.964 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:47 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:51:48.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223666 8.15 MB/sec execute 422 sec latency 221.314 ms 2024-06-11T13:51:48.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: pgmap v522: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-06-11T13:51:48.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:48.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:48.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:48.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:48.865 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: Adjusting osd_memory_target on smithi135 to 2704M 2024-06-11T13:51:48.866 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:48 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:51:49.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: pgmap v522: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-06-11T13:51:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-06-11T13:51:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: Adjusting osd_memory_target on smithi135 to 2704M 2024-06-11T13:51:49.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:48 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:51:49.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223757 8.14 MB/sec execute 423 sec latency 137.644 ms 2024-06-11T13:51:50.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 223913 8.14 MB/sec execute 424 sec latency 154.377 ms 2024-06-11T13:51:50.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:50 smithi069 ceph-mon[25054]: pgmap v523: 97 pgs: 97 active+clean; 337 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 563 op/s 2024-06-11T13:51:50.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:51:50.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:51:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:50 smithi135 ceph-mon[27511]: pgmap v523: 97 pgs: 97 active+clean; 337 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 563 op/s 2024-06-11T13:51:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:51:51.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:51:51.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224021 8.13 MB/sec execute 425 sec latency 218.611 ms 2024-06-11T13:51:52.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224107 8.12 MB/sec execute 426 sec latency 206.286 ms 2024-06-11T13:51:52.916 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:52 smithi069 ceph-mon[25054]: pgmap v524: 97 pgs: 97 active+clean; 337 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-06-11T13:51:53.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:52 smithi135 ceph-mon[27511]: pgmap v524: 97 pgs: 97 active+clean; 337 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-06-11T13:51:53.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224223 8.11 MB/sec execute 427 sec latency 134.820 ms 2024-06-11T13:51:54.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224339 8.10 MB/sec execute 428 sec latency 211.803 ms 2024-06-11T13:51:54.856 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:54 smithi069 ceph-mon[25054]: pgmap v525: 97 pgs: 97 active+clean; 339 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 455 op/s 2024-06-11T13:51:55.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:54 smithi135 ceph-mon[27511]: pgmap v525: 97 pgs: 97 active+clean; 339 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 455 op/s 2024-06-11T13:51:55.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224449 8.08 MB/sec execute 429 sec latency 217.119 ms 2024-06-11T13:51:56.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 224864 8.08 MB/sec execute 430 sec latency 28.317 ms 2024-06-11T13:51:56.772 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:56 smithi069 ceph-mon[25054]: pgmap v526: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.7 MiB/s wr, 381 op/s 2024-06-11T13:51:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:56 smithi135 ceph-mon[27511]: pgmap v526: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.7 MiB/s wr, 381 op/s 2024-06-11T13:51:57.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 225201 8.08 MB/sec execute 431 sec latency 64.615 ms 2024-06-11T13:51:58.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 225464 8.08 MB/sec execute 432 sec latency 157.434 ms 2024-06-11T13:51:58.921 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:51:58 smithi069 ceph-mon[25054]: pgmap v527: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 MiB/s wr, 183 op/s 2024-06-11T13:51:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:51:58 smithi135 ceph-mon[27511]: pgmap v527: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.0 MiB/s wr, 183 op/s 2024-06-11T13:51:59.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 225903 8.10 MB/sec execute 433 sec latency 41.789 ms 2024-06-11T13:52:00.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 226418 8.10 MB/sec execute 434 sec latency 39.785 ms 2024-06-11T13:52:00.826 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:00 smithi069 ceph-mon[25054]: pgmap v528: 97 pgs: 97 active+clean; 339 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.8 MiB/s wr, 257 op/s 2024-06-11T13:52:01.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:00 smithi135 ceph-mon[27511]: pgmap v528: 97 pgs: 97 active+clean; 339 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.8 MiB/s wr, 257 op/s 2024-06-11T13:52:01.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 226805 8.10 MB/sec execute 435 sec latency 41.543 ms 2024-06-11T13:52:02.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 227423 8.13 MB/sec execute 436 sec latency 67.523 ms 2024-06-11T13:52:02.927 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:02 smithi069 ceph-mon[25054]: pgmap v529: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.0 MiB/s wr, 287 op/s 2024-06-11T13:52:03.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:02 smithi135 ceph-mon[27511]: pgmap v529: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.0 MiB/s wr, 287 op/s 2024-06-11T13:52:03.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 227675 8.13 MB/sec execute 437 sec latency 129.670 ms 2024-06-11T13:52:04.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 227819 8.12 MB/sec execute 438 sec latency 215.194 ms 2024-06-11T13:52:04.911 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:04 smithi069 ceph-mon[25054]: pgmap v530: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.1 MiB/s wr, 295 op/s 2024-06-11T13:52:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:04 smithi135 ceph-mon[27511]: pgmap v530: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.1 MiB/s wr, 295 op/s 2024-06-11T13:52:05.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228010 8.12 MB/sec execute 439 sec latency 281.237 ms 2024-06-11T13:52:06.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228163 8.10 MB/sec execute 440 sec latency 300.114 ms 2024-06-11T13:52:06.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:06 smithi069 ceph-mon[25054]: pgmap v531: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-06-11T13:52:07.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:06 smithi135 ceph-mon[27511]: pgmap v531: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-06-11T13:52:07.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228196 8.09 MB/sec execute 441 sec latency 395.651 ms 2024-06-11T13:52:08.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228205 8.07 MB/sec execute 442 sec latency 809.367 ms 2024-06-11T13:52:08.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:08 smithi069 ceph-mon[25054]: pgmap v532: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-06-11T13:52:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:08 smithi135 ceph-mon[27511]: pgmap v532: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-06-11T13:52:09.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228223 8.05 MB/sec execute 443 sec latency 967.918 ms 2024-06-11T13:52:10.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 8.03 MB/sec execute 444 sec latency 1444.734 ms 2024-06-11T13:52:10.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:10 smithi069 ceph-mon[25054]: pgmap v533: 97 pgs: 97 active+clean; 343 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-06-11T13:52:11.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:10 smithi135 ceph-mon[27511]: pgmap v533: 97 pgs: 97 active+clean; 343 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-06-11T13:52:11.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 8.01 MB/sec execute 445 sec latency 1502.177 ms 2024-06-11T13:52:12.299 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 8.00 MB/sec execute 446 sec latency 2502.228 ms 2024-06-11T13:52:12.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:12 smithi069 ceph-mon[25054]: pgmap v534: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.0 MiB/s wr, 412 op/s 2024-06-11T13:52:13.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:12 smithi135 ceph-mon[27511]: pgmap v534: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.0 MiB/s wr, 412 op/s 2024-06-11T13:52:13.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 7.98 MB/sec execute 447 sec latency 3502.259 ms 2024-06-11T13:52:14.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 7.96 MB/sec execute 448 sec latency 4502.316 ms 2024-06-11T13:52:14.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:14 smithi069 ceph-mon[25054]: pgmap v535: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.9 MiB/s wr, 290 op/s 2024-06-11T13:52:15.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:14 smithi135 ceph-mon[27511]: pgmap v535: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.9 MiB/s wr, 290 op/s 2024-06-11T13:52:15.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 7.94 MB/sec execute 449 sec latency 5502.342 ms 2024-06-11T13:52:16.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228234 7.93 MB/sec execute 450 sec latency 6502.368 ms 2024-06-11T13:52:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:16 smithi069 ceph-mon[25054]: pgmap v536: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.8 KiB/s rd, 2.4 MiB/s wr, 262 op/s 2024-06-11T13:52:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:16 smithi135 ceph-mon[27511]: pgmap v536: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.8 KiB/s rd, 2.4 MiB/s wr, 262 op/s 2024-06-11T13:52:17.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.91 MB/sec execute 451 sec latency 7495.998 ms 2024-06-11T13:52:18.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.89 MB/sec execute 452 sec latency 8496.024 ms 2024-06-11T13:52:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:18 smithi069 ceph-mon[25054]: pgmap v537: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.8 KiB/s rd, 562 KiB/s wr, 57 op/s 2024-06-11T13:52:19.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:18 smithi135 ceph-mon[27511]: pgmap v537: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.8 KiB/s rd, 562 KiB/s wr, 57 op/s 2024-06-11T13:52:19.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.87 MB/sec execute 453 sec latency 9496.052 ms 2024-06-11T13:52:20.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.86 MB/sec execute 454 sec latency 10496.078 ms 2024-06-11T13:52:20.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:20 smithi069 ceph-mon[25054]: pgmap v538: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.8 KiB/s rd, 574 KiB/s wr, 58 op/s 2024-06-11T13:52:21.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:20 smithi135 ceph-mon[27511]: pgmap v538: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.8 KiB/s rd, 574 KiB/s wr, 58 op/s 2024-06-11T13:52:21.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.84 MB/sec execute 455 sec latency 11496.105 ms 2024-06-11T13:52:22.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.82 MB/sec execute 456 sec latency 12496.132 ms 2024-06-11T13:52:22.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:22 smithi069 ceph-mon[25054]: pgmap v539: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 201 KiB/s wr, 11 op/s 2024-06-11T13:52:23.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:22 smithi135 ceph-mon[27511]: pgmap v539: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 201 KiB/s wr, 11 op/s 2024-06-11T13:52:23.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.80 MB/sec execute 457 sec latency 13496.192 ms 2024-06-11T13:52:24.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.79 MB/sec execute 458 sec latency 14496.246 ms 2024-06-11T13:52:24.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:24 smithi069 ceph-mon[25054]: pgmap v540: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 104 KiB/s wr, 7 op/s 2024-06-11T13:52:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:24 smithi135 ceph-mon[27511]: pgmap v540: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 104 KiB/s wr, 7 op/s 2024-06-11T13:52:25.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228235 7.77 MB/sec execute 459 sec latency 15496.298 ms 2024-06-11T13:52:26.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.76 MB/sec execute 460 sec latency 15975.459 ms 2024-06-11T13:52:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:26 smithi069 ceph-mon[25054]: pgmap v541: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 79 KiB/s wr, 2 op/s 2024-06-11T13:52:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:26 smithi135 ceph-mon[27511]: pgmap v541: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 79 KiB/s wr, 2 op/s 2024-06-11T13:52:27.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.74 MB/sec execute 461 sec latency 1089.137 ms 2024-06-11T13:52:28.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.73 MB/sec execute 462 sec latency 2089.190 ms 2024-06-11T13:52:28.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:28 smithi069 ceph-mon[25054]: pgmap v542: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 77 KiB/s wr, 2 op/s 2024-06-11T13:52:29.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:28 smithi135 ceph-mon[27511]: pgmap v542: 97 pgs: 97 active+clean; 340 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 77 KiB/s wr, 2 op/s 2024-06-11T13:52:29.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.71 MB/sec execute 463 sec latency 3089.244 ms 2024-06-11T13:52:30.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.69 MB/sec execute 464 sec latency 4089.268 ms 2024-06-11T13:52:30.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:30 smithi069 ceph-mon[25054]: pgmap v543: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 122 KiB/s wr, 14 op/s 2024-06-11T13:52:31.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:30 smithi135 ceph-mon[27511]: pgmap v543: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 122 KiB/s wr, 14 op/s 2024-06-11T13:52:31.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.68 MB/sec execute 465 sec latency 5089.319 ms 2024-06-11T13:52:32.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.66 MB/sec execute 466 sec latency 6089.345 ms 2024-06-11T13:52:32.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:32 smithi069 ceph-mon[25054]: pgmap v544: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 132 KiB/s wr, 14 op/s 2024-06-11T13:52:33.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:32 smithi135 ceph-mon[27511]: pgmap v544: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 132 KiB/s wr, 14 op/s 2024-06-11T13:52:33.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.65 MB/sec execute 467 sec latency 7089.398 ms 2024-06-11T13:52:34.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.63 MB/sec execute 468 sec latency 8089.458 ms 2024-06-11T13:52:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:34 smithi135 ceph-mon[27511]: pgmap v545: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 132 KiB/s wr, 14 op/s 2024-06-11T13:52:35.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:34 smithi069 ceph-mon[25054]: pgmap v545: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 132 KiB/s wr, 14 op/s 2024-06-11T13:52:35.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.61 MB/sec execute 469 sec latency 9089.511 ms 2024-06-11T13:52:36.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.60 MB/sec execute 470 sec latency 10089.572 ms 2024-06-11T13:52:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:36 smithi069 ceph-mon[25054]: pgmap v546: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 133 KiB/s wr, 14 op/s 2024-06-11T13:52:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:36 smithi135 ceph-mon[27511]: pgmap v546: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 133 KiB/s wr, 14 op/s 2024-06-11T13:52:37.300 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228389 7.58 MB/sec execute 471 sec latency 11089.625 ms 2024-06-11T13:52:38.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.57 MB/sec execute 472 sec latency 11568.390 ms 2024-06-11T13:52:38.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:38 smithi069 ceph-mon[25054]: pgmap v547: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 133 KiB/s wr, 14 op/s 2024-06-11T13:52:39.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:38 smithi135 ceph-mon[27511]: pgmap v547: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 133 KiB/s wr, 14 op/s 2024-06-11T13:52:39.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.55 MB/sec execute 473 sec latency 1079.514 ms 2024-06-11T13:52:40.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:39 smithi135 ceph-mon[27511]: Health check failed: Slow OSD heartbeats on back (longest 1390.668ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:52:40.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:39 smithi135 ceph-mon[27511]: Health check failed: Slow OSD heartbeats on front (longest 1390.406ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:52:40.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:39 smithi069 ceph-mon[25054]: Health check failed: Slow OSD heartbeats on back (longest 1390.668ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:52:40.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:39 smithi069 ceph-mon[25054]: Health check failed: Slow OSD heartbeats on front (longest 1390.406ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:52:40.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.54 MB/sec execute 474 sec latency 2079.539 ms 2024-06-11T13:52:41.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:40 smithi135 ceph-mon[27511]: pgmap v548: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 234 KiB/s wr, 26 op/s 2024-06-11T13:52:41.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:40 smithi069 ceph-mon[25054]: pgmap v548: 97 pgs: 97 active+clean; 341 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 234 KiB/s wr, 26 op/s 2024-06-11T13:52:41.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.52 MB/sec execute 475 sec latency 3079.574 ms 2024-06-11T13:52:42.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.50 MB/sec execute 476 sec latency 4079.600 ms 2024-06-11T13:52:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:42 smithi135 ceph-mon[27511]: pgmap v549: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 184 KiB/s wr, 15 op/s 2024-06-11T13:52:43.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:42 smithi069 ceph-mon[25054]: pgmap v549: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 184 KiB/s wr, 15 op/s 2024-06-11T13:52:43.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.49 MB/sec execute 477 sec latency 5079.628 ms 2024-06-11T13:52:44.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.47 MB/sec execute 478 sec latency 6079.654 ms 2024-06-11T13:52:45.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:44 smithi069 ceph-mon[25054]: pgmap v550: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 176 KiB/s wr, 18 op/s 2024-06-11T13:52:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:44 smithi135 ceph-mon[27511]: pgmap v550: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 176 KiB/s wr, 18 op/s 2024-06-11T13:52:45.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.46 MB/sec execute 479 sec latency 7079.706 ms 2024-06-11T13:52:46.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228488 7.44 MB/sec execute 480 sec latency 8079.731 ms 2024-06-11T13:52:46.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:46 smithi069 ceph-mon[25054]: pgmap v551: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 176 KiB/s wr, 18 op/s 2024-06-11T13:52:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:46 smithi135 ceph-mon[27511]: pgmap v551: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 176 KiB/s wr, 18 op/s 2024-06-11T13:52:47.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228493 7.43 MB/sec execute 481 sec latency 9076.755 ms 2024-06-11T13:52:48.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228511 7.41 MB/sec execute 482 sec latency 9470.227 ms 2024-06-11T13:52:48.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:48 smithi069 ceph-mon[25054]: pgmap v552: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 176 KiB/s wr, 18 op/s 2024-06-11T13:52:49.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:48 smithi135 ceph-mon[27511]: pgmap v552: 97 pgs: 97 active+clean; 342 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 176 KiB/s wr, 18 op/s 2024-06-11T13:52:49.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228511 7.40 MB/sec execute 483 sec latency 1576.386 ms 2024-06-11T13:52:50.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228511 7.38 MB/sec execute 484 sec latency 2576.412 ms 2024-06-11T13:52:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:51 smithi135 ceph-mon[27511]: pgmap v553: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 KiB/s rd, 210 KiB/s wr, 23 op/s 2024-06-11T13:52:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:52:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:52:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:52:51.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228511 7.37 MB/sec execute 485 sec latency 3576.440 ms 2024-06-11T13:52:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:51 smithi069 ceph-mon[25054]: pgmap v553: 97 pgs: 97 active+clean; 340 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 1.1 KiB/s rd, 210 KiB/s wr, 23 op/s 2024-06-11T13:52:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:52:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:52:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:52:52.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228511 7.35 MB/sec execute 486 sec latency 4576.493 ms 2024-06-11T13:52:53.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:53 smithi069 ceph-mon[25054]: pgmap v554: 97 pgs: 97 active+clean; 339 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 853 B/s rd, 114 KiB/s wr, 12 op/s 2024-06-11T13:52:53.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:53 smithi135 ceph-mon[27511]: pgmap v554: 97 pgs: 97 active+clean; 339 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 853 B/s rd, 114 KiB/s wr, 12 op/s 2024-06-11T13:52:53.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.34 MB/sec execute 487 sec latency 4975.233 ms 2024-06-11T13:52:54.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.32 MB/sec execute 488 sec latency 1145.001 ms 2024-06-11T13:52:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:55 smithi135 ceph-mon[27511]: pgmap v555: 97 pgs: 97 active+clean; 338 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 97 KiB/s wr, 21 op/s 2024-06-11T13:52:55.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.31 MB/sec execute 489 sec latency 2145.052 ms 2024-06-11T13:52:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:55 smithi069 ceph-mon[25054]: pgmap v555: 97 pgs: 97 active+clean; 338 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 97 KiB/s wr, 21 op/s 2024-06-11T13:52:56.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.29 MB/sec execute 490 sec latency 3145.079 ms 2024-06-11T13:52:56.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:56 smithi069 ceph-mon[25054]: pgmap v556: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 138 KiB/s wr, 21 op/s 2024-06-11T13:52:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:56 smithi135 ceph-mon[27511]: pgmap v556: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 138 KiB/s wr, 21 op/s 2024-06-11T13:52:57.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.28 MB/sec execute 491 sec latency 4145.133 ms 2024-06-11T13:52:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:57 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 3456.894ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:52:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:57 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 3456.745ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:52:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:57 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 3456.894ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:52:58.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:57 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 3456.745ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:52:58.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.26 MB/sec execute 492 sec latency 5145.161 ms 2024-06-11T13:52:58.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:52:58 smithi069 ceph-mon[25054]: pgmap v557: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 138 KiB/s wr, 21 op/s 2024-06-11T13:52:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:52:58 smithi135 ceph-mon[27511]: pgmap v557: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 138 KiB/s wr, 21 op/s 2024-06-11T13:52:59.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.25 MB/sec execute 493 sec latency 6145.217 ms 2024-06-11T13:53:00.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.23 MB/sec execute 494 sec latency 7145.268 ms 2024-06-11T13:53:00.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:00 smithi069 ceph-mon[25054]: pgmap v558: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 149 KiB/s wr, 24 op/s 2024-06-11T13:53:01.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:00 smithi135 ceph-mon[27511]: pgmap v558: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 149 KiB/s wr, 24 op/s 2024-06-11T13:53:01.301 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228610 7.22 MB/sec execute 495 sec latency 8145.333 ms 2024-06-11T13:53:02.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228611 7.20 MB/sec execute 496 sec latency 8471.569 ms 2024-06-11T13:53:03.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:02 smithi135 ceph-mon[27511]: pgmap v559: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 116 KiB/s wr, 20 op/s 2024-06-11T13:53:03.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:02 smithi069 ceph-mon[25054]: pgmap v559: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 116 KiB/s wr, 20 op/s 2024-06-11T13:53:03.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228611 7.19 MB/sec execute 497 sec latency 1691.792 ms 2024-06-11T13:53:04.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228611 7.18 MB/sec execute 498 sec latency 2691.828 ms 2024-06-11T13:53:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:04 smithi135 ceph-mon[27511]: pgmap v560: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 682 B/s rd, 118 KiB/s wr, 20 op/s 2024-06-11T13:53:05.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:04 smithi069 ceph-mon[25054]: pgmap v560: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 682 B/s rd, 118 KiB/s wr, 20 op/s 2024-06-11T13:53:05.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228611 7.16 MB/sec execute 499 sec latency 3691.878 ms 2024-06-11T13:53:06.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228611 7.15 MB/sec execute 500 sec latency 4691.930 ms 2024-06-11T13:53:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:06 smithi069 ceph-mon[25054]: pgmap v561: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 75 KiB/s wr, 11 op/s 2024-06-11T13:53:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:06 smithi135 ceph-mon[27511]: pgmap v561: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 75 KiB/s wr, 11 op/s 2024-06-11T13:53:07.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228611 7.13 MB/sec execute 501 sec latency 5691.983 ms 2024-06-11T13:53:07.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:07 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 4413.151ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:53:07.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:07 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 4413.573ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:53:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:07 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 4413.151ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:53:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:07 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 4413.573ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:53:08.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.12 MB/sec execute 502 sec latency 6131.993 ms 2024-06-11T13:53:08.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:08 smithi069 ceph-mon[25054]: pgmap v562: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 25 KiB/s wr, 7 op/s 2024-06-11T13:53:09.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:08 smithi135 ceph-mon[27511]: pgmap v562: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 25 KiB/s wr, 7 op/s 2024-06-11T13:53:09.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.11 MB/sec execute 503 sec latency 1223.782 ms 2024-06-11T13:53:10.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.09 MB/sec execute 504 sec latency 2223.809 ms 2024-06-11T13:53:11.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:10 smithi135 ceph-mon[27511]: pgmap v563: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 54 KiB/s wr, 14 op/s 2024-06-11T13:53:11.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:10 smithi069 ceph-mon[25054]: pgmap v563: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 54 KiB/s wr, 14 op/s 2024-06-11T13:53:11.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.08 MB/sec execute 505 sec latency 3223.835 ms 2024-06-11T13:53:12.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.07 MB/sec execute 506 sec latency 4223.860 ms 2024-06-11T13:53:13.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:12 smithi135 ceph-mon[27511]: pgmap v564: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 111 KiB/s wr, 14 op/s 2024-06-11T13:53:13.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:12 smithi069 ceph-mon[25054]: pgmap v564: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 111 KiB/s wr, 14 op/s 2024-06-11T13:53:13.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.05 MB/sec execute 507 sec latency 5223.887 ms 2024-06-11T13:53:14.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.04 MB/sec execute 508 sec latency 6223.914 ms 2024-06-11T13:53:15.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:14 smithi135 ceph-mon[27511]: pgmap v565: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 122 KiB/s wr, 16 op/s 2024-06-11T13:53:15.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:14 smithi069 ceph-mon[25054]: pgmap v565: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 122 KiB/s wr, 16 op/s 2024-06-11T13:53:15.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.02 MB/sec execute 509 sec latency 7223.937 ms 2024-06-11T13:53:16.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228685 7.01 MB/sec execute 510 sec latency 8223.960 ms 2024-06-11T13:53:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:16 smithi069 ceph-mon[25054]: pgmap v566: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 114 KiB/s wr, 14 op/s 2024-06-11T13:53:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:16 smithi135 ceph-mon[27511]: pgmap v566: 97 pgs: 97 active+clean; 332 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 114 KiB/s wr, 14 op/s 2024-06-11T13:53:17.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228704 7.00 MB/sec execute 511 sec latency 8643.264 ms 2024-06-11T13:53:18.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228704 6.98 MB/sec execute 512 sec latency 1544.232 ms 2024-06-11T13:53:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:18 smithi069 ceph-mon[25054]: pgmap v567: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 114 KiB/s wr, 13 op/s 2024-06-11T13:53:19.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:18 smithi135 ceph-mon[27511]: pgmap v567: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 114 KiB/s wr, 13 op/s 2024-06-11T13:53:19.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228704 6.97 MB/sec execute 513 sec latency 2544.261 ms 2024-06-11T13:53:20.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228704 6.96 MB/sec execute 514 sec latency 3544.289 ms 2024-06-11T13:53:21.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:20 smithi135 ceph-mon[27511]: pgmap v568: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 119 KiB/s wr, 15 op/s 2024-06-11T13:53:21.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:20 smithi069 ceph-mon[25054]: pgmap v568: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 119 KiB/s wr, 15 op/s 2024-06-11T13:53:21.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228704 6.94 MB/sec execute 515 sec latency 4544.314 ms 2024-06-11T13:53:22.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228704 6.93 MB/sec execute 516 sec latency 5544.338 ms 2024-06-11T13:53:22.894 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:22 smithi069 ceph-mon[25054]: pgmap v569: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 90 KiB/s wr, 8 op/s 2024-06-11T13:53:23.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:22 smithi135 ceph-mon[27511]: pgmap v569: 97 pgs: 97 active+clean; 332 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 90 KiB/s wr, 8 op/s 2024-06-11T13:53:23.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.92 MB/sec execute 517 sec latency 5921.240 ms 2024-06-11T13:53:24.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.90 MB/sec execute 518 sec latency 1344.342 ms 2024-06-11T13:53:25.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:24 smithi135 ceph-mon[27511]: pgmap v570: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 76 KiB/s wr, 12 op/s 2024-06-11T13:53:25.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:24 smithi069 ceph-mon[25054]: pgmap v570: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 76 KiB/s wr, 12 op/s 2024-06-11T13:53:25.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.89 MB/sec execute 519 sec latency 2344.369 ms 2024-06-11T13:53:26.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.88 MB/sec execute 520 sec latency 3344.393 ms 2024-06-11T13:53:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:26 smithi069 ceph-mon[25054]: pgmap v571: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 94 KiB/s wr, 12 op/s 2024-06-11T13:53:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:26 smithi135 ceph-mon[27511]: pgmap v571: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 94 KiB/s wr, 12 op/s 2024-06-11T13:53:27.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.87 MB/sec execute 521 sec latency 4344.430 ms 2024-06-11T13:53:28.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.85 MB/sec execute 522 sec latency 5344.480 ms 2024-06-11T13:53:29.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:28 smithi069 ceph-mon[25054]: pgmap v572: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 94 KiB/s wr, 12 op/s 2024-06-11T13:53:29.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:28 smithi069 ceph-mon[25054]: pgmap v573: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 298 B/s rd, 110 KiB/s wr, 13 op/s 2024-06-11T13:53:29.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:53:29.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:53:29.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:28 smithi069 ceph-mon[25054]: Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-11T13:53:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:28 smithi135 ceph-mon[27511]: pgmap v572: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 94 KiB/s wr, 12 op/s 2024-06-11T13:53:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:28 smithi135 ceph-mon[27511]: pgmap v573: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 298 B/s rd, 110 KiB/s wr, 13 op/s 2024-06-11T13:53:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:53:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:53:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:28 smithi135 ceph-mon[27511]: Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-11T13:53:29.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.84 MB/sec execute 523 sec latency 6344.517 ms 2024-06-11T13:53:30.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.83 MB/sec execute 524 sec latency 7344.542 ms 2024-06-11T13:53:31.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:30 smithi069 ceph-mon[25054]: pgmap v574: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 298 B/s rd, 98 KiB/s wr, 11 op/s 2024-06-11T13:53:31.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:30 smithi135 ceph-mon[27511]: pgmap v574: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 298 B/s rd, 98 KiB/s wr, 11 op/s 2024-06-11T13:53:31.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.81 MB/sec execute 525 sec latency 8344.569 ms 2024-06-11T13:53:32.302 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.80 MB/sec execute 526 sec latency 9344.620 ms 2024-06-11T13:53:33.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:33 smithi135 ceph-mon[27511]: pgmap v575: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 298 B/s rd, 98 KiB/s wr, 11 op/s 2024-06-11T13:53:33.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.79 MB/sec execute 527 sec latency 10344.680 ms 2024-06-11T13:53:33.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:33 smithi069 ceph-mon[25054]: pgmap v575: 97 pgs: 97 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 298 B/s rd, 98 KiB/s wr, 11 op/s 2024-06-11T13:53:34.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228744 6.77 MB/sec execute 528 sec latency 11344.707 ms 2024-06-11T13:53:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:35 smithi135 ceph-mon[27511]: pgmap v576: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 198 B/s rd, 36 KiB/s wr, 3 op/s 2024-06-11T13:53:35.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.76 MB/sec execute 529 sec latency 12337.952 ms 2024-06-11T13:53:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:35 smithi069 ceph-mon[25054]: pgmap v576: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 198 B/s rd, 36 KiB/s wr, 3 op/s 2024-06-11T13:53:36.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:36 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:53:36.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.75 MB/sec execute 530 sec latency 13337.987 ms 2024-06-11T13:53:36.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:36 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:53:37.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:37 smithi135 ceph-mon[27511]: pgmap v577: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 B/s rd, 28 KiB/s wr, 1 op/s 2024-06-11T13:53:37.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.74 MB/sec execute 531 sec latency 14338.015 ms 2024-06-11T13:53:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:37 smithi069 ceph-mon[25054]: pgmap v577: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 B/s rd, 28 KiB/s wr, 1 op/s 2024-06-11T13:53:38.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.72 MB/sec execute 532 sec latency 15338.054 ms 2024-06-11T13:53:39.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:39 smithi135 ceph-mon[27511]: pgmap v578: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 B/s rd, 28 KiB/s wr, 1 op/s 2024-06-11T13:53:39.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.71 MB/sec execute 533 sec latency 16338.083 ms 2024-06-11T13:53:39.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:39 smithi069 ceph-mon[25054]: pgmap v578: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 198 B/s rd, 28 KiB/s wr, 1 op/s 2024-06-11T13:53:40.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.70 MB/sec execute 534 sec latency 17338.123 ms 2024-06-11T13:53:41.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:41 smithi135 ceph-mon[27511]: pgmap v579: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 24 KiB/s wr, 2 op/s 2024-06-11T13:53:41.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.69 MB/sec execute 535 sec latency 18338.161 ms 2024-06-11T13:53:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:41 smithi069 ceph-mon[25054]: pgmap v579: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 24 KiB/s wr, 2 op/s 2024-06-11T13:53:42.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.67 MB/sec execute 536 sec latency 19338.217 ms 2024-06-11T13:53:43.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:43 smithi135 ceph-mon[27511]: pgmap v580: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 1 op/s 2024-06-11T13:53:43.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.66 MB/sec execute 537 sec latency 20338.294 ms 2024-06-11T13:53:43.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:43 smithi069 ceph-mon[25054]: pgmap v580: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 1 op/s 2024-06-11T13:53:44.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.65 MB/sec execute 538 sec latency 21338.320 ms 2024-06-11T13:53:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:45 smithi135 ceph-mon[27511]: pgmap v581: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 28 KiB/s wr, 3 op/s 2024-06-11T13:53:45.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.64 MB/sec execute 539 sec latency 22338.349 ms 2024-06-11T13:53:45.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:45 smithi069 ceph-mon[25054]: pgmap v581: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 28 KiB/s wr, 3 op/s 2024-06-11T13:53:46.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.62 MB/sec execute 540 sec latency 23338.377 ms 2024-06-11T13:53:46.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:46 smithi069 ceph-mon[25054]: pgmap v582: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 27 KiB/s wr, 2 op/s 2024-06-11T13:53:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:46 smithi135 ceph-mon[27511]: pgmap v582: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 27 KiB/s wr, 2 op/s 2024-06-11T13:53:47.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.61 MB/sec execute 541 sec latency 24338.433 ms 2024-06-11T13:53:48.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.60 MB/sec execute 542 sec latency 25338.484 ms 2024-06-11T13:53:49.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228746 6.59 MB/sec execute 543 sec latency 26338.552 ms 2024-06-11T13:53:49.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:49 smithi069 ceph-mon[25054]: pgmap v583: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.6 KiB/s wr, 2 op/s 2024-06-11T13:53:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:49 smithi135 ceph-mon[27511]: pgmap v583: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.6 KiB/s wr, 2 op/s 2024-06-11T13:53:50.053 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 13:53:49 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T13:53:49.757+0000 7fa83e77a700 -1 osd.1 81 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T13:53:22.643926+0000 front 2024-06-11T13:53:22.644616+0000 (oldest deadline 2024-06-11T13:53:48.958682+0000) 2024-06-11T13:53:50.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228794 6.58 MB/sec execute 544 sec latency 26818.590 ms 2024-06-11T13:53:50.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:50 smithi069 ceph-mon[25054]: osd.0 reported failed by osd.1 2024-06-11T13:53:50.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:50 smithi069 ceph-mon[25054]: osd.0 failure report canceled by osd.1 2024-06-11T13:53:50.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:50 smithi135 ceph-mon[27511]: osd.0 reported failed by osd.1 2024-06-11T13:53:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:50 smithi135 ceph-mon[27511]: osd.0 failure report canceled by osd.1 2024-06-11T13:53:51.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228794 6.56 MB/sec execute 545 sec latency 1366.245 ms 2024-06-11T13:53:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:51 smithi069 ceph-mon[25054]: pgmap v584: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.6 KiB/s wr, 2 op/s 2024-06-11T13:53:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:53:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:53:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:51 smithi135 ceph-mon[27511]: pgmap v584: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.6 KiB/s wr, 2 op/s 2024-06-11T13:53:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:53:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:53:52.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228794 6.55 MB/sec execute 546 sec latency 2366.273 ms 2024-06-11T13:53:53.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228794 6.54 MB/sec execute 547 sec latency 3366.310 ms 2024-06-11T13:53:53.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:53 smithi069 ceph-mon[25054]: pgmap v585: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.8 KiB/s wr, 2 op/s 2024-06-11T13:53:53.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:53 smithi135 ceph-mon[27511]: pgmap v585: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.8 KiB/s wr, 2 op/s 2024-06-11T13:53:54.303 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228794 6.53 MB/sec execute 548 sec latency 4366.337 ms 2024-06-11T13:53:55.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228794 6.52 MB/sec execute 549 sec latency 5366.389 ms 2024-06-11T13:53:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:55 smithi069 ceph-mon[25054]: pgmap v586: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 46 KiB/s wr, 6 op/s 2024-06-11T13:53:55.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:55 smithi135 ceph-mon[27511]: pgmap v586: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 46 KiB/s wr, 6 op/s 2024-06-11T13:53:56.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228796 6.50 MB/sec execute 550 sec latency 6362.805 ms 2024-06-11T13:53:56.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:56 smithi069 ceph-mon[25054]: pgmap v587: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 56 KiB/s wr, 7 op/s 2024-06-11T13:53:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:56 smithi135 ceph-mon[27511]: pgmap v587: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 56 KiB/s wr, 7 op/s 2024-06-11T13:53:57.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228796 6.49 MB/sec execute 551 sec latency 7362.857 ms 2024-06-11T13:53:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:57 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 4738.483ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:53:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:57 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 4738.421ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:53:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:57 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 4738.483ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:53:58.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:57 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 4738.421ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:53:58.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228796 6.48 MB/sec execute 552 sec latency 8362.893 ms 2024-06-11T13:53:58.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:53:58 smithi069 ceph-mon[25054]: pgmap v588: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 56 KiB/s wr, 6 op/s 2024-06-11T13:53:59.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:53:58 smithi135 ceph-mon[27511]: pgmap v588: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 56 KiB/s wr, 6 op/s 2024-06-11T13:53:59.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228809 6.47 MB/sec execute 553 sec latency 8862.574 ms 2024-06-11T13:54:00.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228809 6.46 MB/sec execute 554 sec latency 1453.974 ms 2024-06-11T13:54:01.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228809 6.45 MB/sec execute 555 sec latency 2454.037 ms 2024-06-11T13:54:01.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:01 smithi069 ceph-mon[25054]: pgmap v589: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 59 KiB/s wr, 7 op/s 2024-06-11T13:54:01.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:01 smithi135 ceph-mon[27511]: pgmap v589: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 59 KiB/s wr, 7 op/s 2024-06-11T13:54:02.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228809 6.43 MB/sec execute 556 sec latency 3454.090 ms 2024-06-11T13:54:03.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228809 6.42 MB/sec execute 557 sec latency 4454.118 ms 2024-06-11T13:54:03.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:03 smithi069 ceph-mon[25054]: pgmap v590: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 65 KiB/s wr, 8 op/s 2024-06-11T13:54:03.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:03 smithi135 ceph-mon[27511]: pgmap v590: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 65 KiB/s wr, 8 op/s 2024-06-11T13:54:04.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228809 6.41 MB/sec execute 558 sec latency 5454.146 ms 2024-06-11T13:54:05.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:05 smithi069 ceph-mon[25054]: pgmap v591: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 74 KiB/s wr, 11 op/s 2024-06-11T13:54:05.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.41 MB/sec execute 559 sec latency 6017.715 ms 2024-06-11T13:54:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:05 smithi135 ceph-mon[27511]: pgmap v591: 97 pgs: 1 active+clean+laggy, 96 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 74 KiB/s wr, 11 op/s 2024-06-11T13:54:06.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.39 MB/sec execute 560 sec latency 1157.252 ms 2024-06-11T13:54:06.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:06 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 5572.620ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:54:06.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:06 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 5574.294ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:54:06.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:06 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 5572.620ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:54:06.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:06 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 5574.294ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:54:07.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.38 MB/sec execute 561 sec latency 2157.282 ms 2024-06-11T13:54:07.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:07 smithi069 ceph-mon[25054]: pgmap v592: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 49 KiB/s wr, 7 op/s 2024-06-11T13:54:07.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:07 smithi135 ceph-mon[27511]: pgmap v592: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 49 KiB/s wr, 7 op/s 2024-06-11T13:54:08.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.37 MB/sec execute 562 sec latency 3157.308 ms 2024-06-11T13:54:09.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.36 MB/sec execute 563 sec latency 4157.344 ms 2024-06-11T13:54:09.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:09 smithi069 ceph-mon[25054]: pgmap v593: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 35 KiB/s wr, 5 op/s 2024-06-11T13:54:09.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:09 smithi135 ceph-mon[27511]: pgmap v593: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 35 KiB/s wr, 5 op/s 2024-06-11T13:54:10.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.35 MB/sec execute 564 sec latency 5157.371 ms 2024-06-11T13:54:11.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.34 MB/sec execute 565 sec latency 6157.398 ms 2024-06-11T13:54:11.349 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:11 smithi135 ceph-mon[27511]: pgmap v594: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 71 KiB/s wr, 11 op/s 2024-06-11T13:54:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:11 smithi069 ceph-mon[25054]: pgmap v594: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 71 KiB/s wr, 11 op/s 2024-06-11T13:54:12.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.33 MB/sec execute 566 sec latency 7157.428 ms 2024-06-11T13:54:13.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.32 MB/sec execute 567 sec latency 8157.456 ms 2024-06-11T13:54:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:13 smithi069 ceph-mon[25054]: pgmap v595: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 68 KiB/s wr, 10 op/s 2024-06-11T13:54:13.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:13 smithi135 ceph-mon[27511]: pgmap v595: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 68 KiB/s wr, 10 op/s 2024-06-11T13:54:14.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.30 MB/sec execute 568 sec latency 9157.483 ms 2024-06-11T13:54:14.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:14 smithi069 ceph-mon[25054]: pgmap v596: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 62 KiB/s wr, 9 op/s 2024-06-11T13:54:14.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:14 smithi135 ceph-mon[27511]: pgmap v596: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 62 KiB/s wr, 9 op/s 2024-06-11T13:54:15.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.29 MB/sec execute 569 sec latency 10157.511 ms 2024-06-11T13:54:16.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.28 MB/sec execute 570 sec latency 11157.541 ms 2024-06-11T13:54:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:16 smithi069 ceph-mon[25054]: pgmap v597: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 53 KiB/s wr, 6 op/s 2024-06-11T13:54:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:16 smithi135 ceph-mon[27511]: pgmap v597: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 53 KiB/s wr, 6 op/s 2024-06-11T13:54:17.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.27 MB/sec execute 571 sec latency 12157.568 ms 2024-06-11T13:54:18.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.26 MB/sec execute 572 sec latency 13157.597 ms 2024-06-11T13:54:19.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228862 6.25 MB/sec execute 573 sec latency 14157.628 ms 2024-06-11T13:54:19.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:19 smithi069 ceph-mon[25054]: pgmap v598: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 36 KiB/s wr, 5 op/s 2024-06-11T13:54:19.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:19 smithi135 ceph-mon[27511]: pgmap v598: 97 pgs: 97 active+clean; 333 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 36 KiB/s wr, 5 op/s 2024-06-11T13:54:20.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228864 6.24 MB/sec execute 574 sec latency 14518.520 ms 2024-06-11T13:54:21.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228864 6.23 MB/sec execute 575 sec latency 1643.309 ms 2024-06-11T13:54:21.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:21 smithi069 ceph-mon[25054]: pgmap v599: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 51 KiB/s wr, 7 op/s 2024-06-11T13:54:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:21 smithi135 ceph-mon[27511]: pgmap v599: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 51 KiB/s wr, 7 op/s 2024-06-11T13:54:22.304 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228864 6.22 MB/sec execute 576 sec latency 2643.363 ms 2024-06-11T13:54:23.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228898 6.21 MB/sec execute 577 sec latency 3643.393 ms 2024-06-11T13:54:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:23 smithi069 ceph-mon[25054]: pgmap v600: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 16 KiB/s wr, 2 op/s 2024-06-11T13:54:23.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:23 smithi135 ceph-mon[27511]: pgmap v600: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 16 KiB/s wr, 2 op/s 2024-06-11T13:54:24.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228898 6.20 MB/sec execute 578 sec latency 4643.429 ms 2024-06-11T13:54:25.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228898 6.19 MB/sec execute 579 sec latency 5643.487 ms 2024-06-11T13:54:25.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:25 smithi069 ceph-mon[25054]: pgmap v601: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 55 KiB/s wr, 7 op/s 2024-06-11T13:54:25.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:25 smithi135 ceph-mon[27511]: pgmap v601: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 55 KiB/s wr, 7 op/s 2024-06-11T13:54:26.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228898 6.18 MB/sec execute 580 sec latency 6643.552 ms 2024-06-11T13:54:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:26 smithi069 ceph-mon[25054]: pgmap v602: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 110 KiB/s wr, 8 op/s 2024-06-11T13:54:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:26 smithi135 ceph-mon[27511]: pgmap v602: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 110 KiB/s wr, 8 op/s 2024-06-11T13:54:27.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228898 6.17 MB/sec execute 581 sec latency 7643.615 ms 2024-06-11T13:54:28.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:54:28.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:54:28.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:28 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:54:28.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228898 6.15 MB/sec execute 582 sec latency 8643.639 ms 2024-06-11T13:54:28.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:54:28.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:54:28.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:28 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:54:29.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228899 6.14 MB/sec execute 583 sec latency 9643.676 ms 2024-06-11T13:54:29.395 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:29 smithi135 ceph-mon[27511]: pgmap v603: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 110 KiB/s wr, 8 op/s 2024-06-11T13:54:29.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:29 smithi069 ceph-mon[25054]: pgmap v603: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 110 KiB/s wr, 8 op/s 2024-06-11T13:54:30.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228899 6.13 MB/sec execute 584 sec latency 10643.701 ms 2024-06-11T13:54:31.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:31 smithi135 ceph-mon[27511]: pgmap v604: 97 pgs: 3 active+clean+laggy, 94 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 121 KiB/s wr, 9 op/s 2024-06-11T13:54:31.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:31 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:54:31.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228899 6.12 MB/sec execute 585 sec latency 11643.731 ms 2024-06-11T13:54:31.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:31 smithi069 ceph-mon[25054]: pgmap v604: 97 pgs: 3 active+clean+laggy, 94 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 426 B/s rd, 121 KiB/s wr, 9 op/s 2024-06-11T13:54:31.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:31 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:54:32.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228899 6.11 MB/sec execute 586 sec latency 12643.816 ms 2024-06-11T13:54:33.305 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228899 6.10 MB/sec execute 587 sec latency 13643.885 ms 2024-06-11T13:54:33.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:33 smithi069 ceph-mon[25054]: pgmap v605: 97 pgs: 3 active+clean+laggy, 94 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 106 KiB/s wr, 7 op/s 2024-06-11T13:54:33.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:33 smithi135 ceph-mon[27511]: pgmap v605: 97 pgs: 3 active+clean+laggy, 94 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 106 KiB/s wr, 7 op/s 2024-06-11T13:54:34.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 228922 6.09 MB/sec execute 588 sec latency 14275.850 ms 2024-06-11T13:54:35.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229057 6.09 MB/sec execute 589 sec latency 223.070 ms 2024-06-11T13:54:35.374 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:35 smithi069 ceph-mon[25054]: pgmap v606: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 8 op/s 2024-06-11T13:54:35.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:35 smithi135 ceph-mon[27511]: pgmap v606: 97 pgs: 97 active+clean; 334 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 108 KiB/s wr, 8 op/s 2024-06-11T13:54:36.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229285 6.10 MB/sec execute 590 sec latency 105.224 ms 2024-06-11T13:54:36.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:36 smithi069 ceph-mon[25054]: pgmap v607: 97 pgs: 97 active+clean; 335 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 209 KiB/s wr, 5 op/s 2024-06-11T13:54:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:36 smithi135 ceph-mon[27511]: pgmap v607: 97 pgs: 97 active+clean; 335 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 209 KiB/s wr, 5 op/s 2024-06-11T13:54:37.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229568 6.10 MB/sec execute 591 sec latency 69.878 ms 2024-06-11T13:54:38.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 229909 6.09 MB/sec execute 592 sec latency 96.213 ms 2024-06-11T13:54:39.214 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:39 smithi069 ceph-mon[25054]: pgmap v608: 97 pgs: 97 active+clean; 335 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 155 KiB/s wr, 5 op/s 2024-06-11T13:54:39.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230176 6.09 MB/sec execute 593 sec latency 81.426 ms 2024-06-11T13:54:39.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:39 smithi135 ceph-mon[27511]: pgmap v608: 97 pgs: 97 active+clean; 335 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 597 B/s rd, 155 KiB/s wr, 5 op/s 2024-06-11T13:54:40.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230410 6.10 MB/sec execute 594 sec latency 89.090 ms 2024-06-11T13:54:41.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:41 smithi069 ceph-mon[25054]: pgmap v609: 97 pgs: 97 active+clean; 336 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 115 op/s 2024-06-11T13:54:41.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230694 6.10 MB/sec execute 595 sec latency 58.078 ms 2024-06-11T13:54:41.348 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:41 smithi135 ceph-mon[27511]: pgmap v609: 97 pgs: 97 active+clean; 336 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 767 B/s rd, 1.1 MiB/s wr, 115 op/s 2024-06-11T13:54:42.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 230997 6.11 MB/sec execute 596 sec latency 66.691 ms 2024-06-11T13:54:43.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231227 6.11 MB/sec execute 597 sec latency 143.388 ms 2024-06-11T13:54:43.324 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:43 smithi069 ceph-mon[25054]: pgmap v610: 97 pgs: 97 active+clean; 336 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 MiB/s wr, 122 op/s 2024-06-11T13:54:43.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:43 smithi135 ceph-mon[27511]: pgmap v610: 97 pgs: 97 active+clean; 336 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.2 MiB/s wr, 122 op/s 2024-06-11T13:54:44.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231456 6.12 MB/sec execute 598 sec latency 68.931 ms 2024-06-11T13:54:45.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:45 smithi069 ceph-mon[25054]: pgmap v611: 97 pgs: 97 active+clean; 343 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.1 MiB/s wr, 232 op/s 2024-06-11T13:54:45.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 231614 6.11 MB/sec execute 599 sec latency 108.569 ms 2024-06-11T13:54:45.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:45 smithi135 ceph-mon[27511]: pgmap v611: 97 pgs: 97 active+clean; 343 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.1 MiB/s wr, 232 op/s 2024-06-11T13:54:46.312 INFO:teuthology.orchestra.run.smithi069.stdout: 5 cleanup 600 sec 2024-06-11T13:54:46.571 INFO:teuthology.orchestra.run.smithi069.stdout: 0 cleanup 600 sec 2024-06-11T13:54:46.571 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:54:46.571 INFO:teuthology.orchestra.run.smithi069.stdout: Operation Count AvgLat MaxLat 2024-06-11T13:54:46.571 INFO:teuthology.orchestra.run.smithi069.stdout: ---------------------------------------- 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: NTCreateX 117948 5.587 24648.480 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Close 86531 0.133 6.678 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Rename 4986 14.970 26818.579 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Unlink 23867 10.775 25885.409 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Qpathinfo 106889 1.532 24453.964 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Qfileinfo 18591 0.007 0.910 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Qfsinfo 19569 0.585 11.447 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Sfileinfo 9584 18.631 24468.068 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: Find 41254 0.963 18.763 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: WriteX 58135 27.518 87004.596 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: ReadX 184393 0.010 3.513 2024-06-11T13:54:46.572 INFO:teuthology.orchestra.run.smithi069.stdout: LockX 380 0.381 2.807 2024-06-11T13:54:46.573 INFO:teuthology.orchestra.run.smithi069.stdout: UnlockX 380 0.339 7.017 2024-06-11T13:54:46.573 INFO:teuthology.orchestra.run.smithi069.stdout: Flush 8250 0.002 0.042 2024-06-11T13:54:46.573 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:54:46.573 INFO:teuthology.orchestra.run.smithi069.stdout:Throughput 6.11026 MB/sec 5 clients 5 procs max_latency=87004.604 ms 2024-06-11T13:54:46.574 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-06-11T13:54:46.688 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-06-11T13:54:46.783 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-06-11T13:54:46.807 INFO:teuthology.orchestra.run.smithi069.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-06-11T13:54:46.807 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:54:46.807 INFO:teuthology.orchestra.run.smithi069.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-06-11T13:54:46.807 INFO:teuthology.orchestra.run.smithi069.stdout:3 of 5 processes prepared for launch 0 sec 2024-06-11T13:54:46.908 INFO:teuthology.orchestra.run.smithi069.stdout:5 of 5 processes prepared for launch 0 sec 2024-06-11T13:54:46.908 INFO:teuthology.orchestra.run.smithi069.stdout:releasing clients 2024-06-11T13:54:46.909 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:46 smithi069 ceph-mon[25054]: pgmap v612: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.0 MiB/s wr, 300 op/s 2024-06-11T13:54:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:46 smithi135 ceph-mon[27511]: pgmap v612: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.0 MiB/s wr, 300 op/s 2024-06-11T13:54:48.908 INFO:teuthology.orchestra.run.smithi069.stdout: 5 225 13.27 MB/sec execute 1 sec latency 491.509 ms 2024-06-11T13:54:49.329 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:49 smithi069 ceph-mon[25054]: pgmap v613: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.9 MiB/s wr, 298 op/s 2024-06-11T13:54:49.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:49 smithi135 ceph-mon[27511]: pgmap v613: 97 pgs: 97 active+clean; 338 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.9 MiB/s wr, 298 op/s 2024-06-11T13:54:49.908 INFO:teuthology.orchestra.run.smithi069.stdout: 5 338 15.35 MB/sec execute 2 sec latency 195.224 ms 2024-06-11T13:54:50.229 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:50 smithi069 ceph-mon[25054]: pgmap v614: 97 pgs: 97 active+clean; 309 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 7.0 MiB/s wr, 427 op/s 2024-06-11T13:54:50.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:54:50.230 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:50 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:54:50.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:50 smithi135 ceph-mon[27511]: pgmap v614: 97 pgs: 97 active+clean; 309 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 7.0 MiB/s wr, 427 op/s 2024-06-11T13:54:50.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:54:50.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:50 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:54:50.908 INFO:teuthology.orchestra.run.smithi069.stdout: 5 463 16.91 MB/sec execute 3 sec latency 87.761 ms 2024-06-11T13:54:51.908 INFO:teuthology.orchestra.run.smithi069.stdout: 5 659 18.36 MB/sec execute 4 sec latency 182.494 ms 2024-06-11T13:54:52.908 INFO:teuthology.orchestra.run.smithi069.stdout: 5 cleanup 5 sec 2024-06-11T13:54:53.071 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:53 smithi069 ceph-mon[25054]: pgmap v615: 97 pgs: 97 active+clean; 304 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.8 MiB/s wr, 327 op/s 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: 0 cleanup 5 sec 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Operation Count AvgLat MaxLat 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: ---------------------------------------- 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: NTCreateX 505 13.319 190.524 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Close 479 17.174 261.268 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Rename 13 30.311 48.259 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Unlink 31 4.933 32.894 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Qpathinfo 95 1.334 28.879 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Qfileinfo 335 20.253 195.217 2024-06-11T13:54:53.091 INFO:teuthology.orchestra.run.smithi069.stdout: Qfsinfo 75 0.427 2.211 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout: Sfileinfo 61 17.166 36.359 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout: Find 84 0.735 1.441 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout: WriteX 1622 0.028 0.107 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout: ReadX 310 0.017 0.365 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout: Flush 94 16.373 57.406 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T13:54:53.092 INFO:teuthology.orchestra.run.smithi069.stdout:Throughput 18.3567 MB/sec 5 clients 5 procs max_latency=491.509 ms 2024-06-11T13:54:53.093 DEBUG:teuthology.parallel:result is None 2024-06-11T13:54:53.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:53 smithi135 ceph-mon[27511]: pgmap v615: 97 pgs: 97 active+clean; 304 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.8 MiB/s wr, 327 op/s 2024-06-11T13:54:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:55 smithi069 ceph-mon[25054]: pgmap v616: 97 pgs: 97 active+clean; 300 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 8.4 MiB/s wr, 385 op/s 2024-06-11T13:54:55.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:55 smithi135 ceph-mon[27511]: pgmap v616: 97 pgs: 97 active+clean; 300 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 8.4 MiB/s wr, 385 op/s 2024-06-11T13:54:56.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:56 smithi069 ceph-mon[25054]: pgmap v617: 97 pgs: 97 active+clean; 283 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 11 MiB/s wr, 367 op/s 2024-06-11T13:54:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:56 smithi135 ceph-mon[27511]: pgmap v617: 97 pgs: 97 active+clean; 283 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 11 MiB/s wr, 367 op/s 2024-06-11T13:54:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:54:59 smithi069 ceph-mon[25054]: pgmap v618: 97 pgs: 97 active+clean; 279 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.9 MiB/s wr, 299 op/s 2024-06-11T13:54:59.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:54:59 smithi135 ceph-mon[27511]: pgmap v618: 97 pgs: 97 active+clean; 279 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 9.9 MiB/s wr, 299 op/s 2024-06-11T13:55:01.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:01 smithi069 ceph-mon[25054]: pgmap v619: 97 pgs: 97 active+clean; 230 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 9.9 MiB/s wr, 310 op/s 2024-06-11T13:55:01.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:01 smithi135 ceph-mon[27511]: pgmap v619: 97 pgs: 97 active+clean; 230 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 9.9 MiB/s wr, 310 op/s 2024-06-11T13:55:02.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:02 smithi069 ceph-mon[25054]: pgmap v620: 97 pgs: 97 active+clean; 230 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 181 op/s 2024-06-11T13:55:02.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:02 smithi135 ceph-mon[27511]: pgmap v620: 97 pgs: 97 active+clean; 230 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 181 op/s 2024-06-11T13:55:03.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:03 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 5342.595ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:55:03.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:03 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 5340.670ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:55:03.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:03 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 5342.595ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:55:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:03 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 5340.670ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:55:04.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:04 smithi069 ceph-mon[25054]: pgmap v621: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.0 MiB/s wr, 171 op/s 2024-06-11T13:55:04.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:04 smithi135 ceph-mon[27511]: pgmap v621: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.0 MiB/s wr, 171 op/s 2024-06-11T13:55:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:06 smithi069 ceph-mon[25054]: pgmap v622: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.3 MiB/s wr, 107 op/s 2024-06-11T13:55:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:06 smithi135 ceph-mon[27511]: pgmap v622: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.3 MiB/s wr, 107 op/s 2024-06-11T13:55:09.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:09 smithi069 ceph-mon[25054]: pgmap v623: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 26 KiB/s wr, 15 op/s 2024-06-11T13:55:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:09 smithi135 ceph-mon[27511]: pgmap v623: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3.2 KiB/s rd, 26 KiB/s wr, 15 op/s 2024-06-11T13:55:11.349 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:11 smithi135 ceph-mon[27511]: pgmap v624: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 23 KiB/s wr, 14 op/s 2024-06-11T13:55:11.349 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:11 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 2894.589ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:55:11.349 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:11 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 3315.392ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:55:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:11 smithi069 ceph-mon[25054]: pgmap v624: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 511 B/s rd, 23 KiB/s wr, 14 op/s 2024-06-11T13:55:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:11 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 2894.589ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:55:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:11 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 3315.392ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:55:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:13 smithi069 ceph-mon[25054]: pgmap v625: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.0 KiB/s wr, 2 op/s 2024-06-11T13:55:13.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:13 smithi135 ceph-mon[27511]: pgmap v625: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 4.0 KiB/s wr, 2 op/s 2024-06-11T13:55:15.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:15 smithi069 ceph-mon[25054]: pgmap v626: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 3.8 KiB/s wr, 2 op/s 2024-06-11T13:55:15.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:15 smithi135 ceph-mon[27511]: pgmap v626: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 3.8 KiB/s wr, 2 op/s 2024-06-11T13:55:16.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:16 smithi069 ceph-mon[25054]: pgmap v627: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 4.2 KiB/s wr, 1 op/s 2024-06-11T13:55:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:16 smithi135 ceph-mon[27511]: pgmap v627: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 4.2 KiB/s wr, 1 op/s 2024-06-11T13:55:19.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:19 smithi069 ceph-mon[25054]: pgmap v628: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:19.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:19 smithi135 ceph-mon[27511]: pgmap v628: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:21.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:21 smithi069 ceph-mon[25054]: pgmap v629: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:21.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:21 smithi135 ceph-mon[27511]: pgmap v629: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:23 smithi069 ceph-mon[25054]: pgmap v630: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:23 smithi135 ceph-mon[27511]: pgmap v630: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:25.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:25 smithi069 ceph-mon[25054]: pgmap v631: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:25.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:25 smithi135 ceph-mon[27511]: pgmap v631: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:26 smithi069 ceph-mon[25054]: pgmap v632: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:26 smithi135 ceph-mon[27511]: pgmap v632: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s 2024-06-11T13:55:27.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:27 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 2678.368ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:55:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:27 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 2678.368ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:55:28.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:28 smithi069 ceph-mon[25054]: pgmap v633: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:29.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:28 smithi135 ceph-mon[27511]: pgmap v633: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:31.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:31 smithi069 ceph-mon[25054]: pgmap v634: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:55:31.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:31 smithi135 ceph-mon[27511]: pgmap v634: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:55:33.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:33 smithi069 ceph-mon[25054]: pgmap v635: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:33.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:33 smithi135 ceph-mon[27511]: pgmap v635: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:35 smithi069 ceph-mon[25054]: pgmap v636: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:35.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:35 smithi135 ceph-mon[27511]: pgmap v636: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:36 smithi069 ceph-mon[25054]: pgmap v637: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:36 smithi135 ceph-mon[27511]: pgmap v637: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:39.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:39 smithi069 ceph-mon[25054]: pgmap v638: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:39.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:39 smithi135 ceph-mon[27511]: pgmap v638: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:41.349 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:41 smithi135 ceph-mon[27511]: pgmap v639: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:55:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:41 smithi069 ceph-mon[25054]: pgmap v639: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:55:43.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:43 smithi069 ceph-mon[25054]: pgmap v640: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:43.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:43 smithi135 ceph-mon[27511]: pgmap v640: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:45.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:45 smithi069 ceph-mon[25054]: pgmap v641: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:45.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:45 smithi135 ceph-mon[27511]: pgmap v641: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:46.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:46 smithi069 ceph-mon[25054]: pgmap v642: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:46 smithi135 ceph-mon[27511]: pgmap v642: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:49.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:49 smithi069 ceph-mon[25054]: pgmap v643: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:49.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:49 smithi135 ceph-mon[27511]: pgmap v643: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:51 smithi069 ceph-mon[25054]: pgmap v644: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:55:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:55:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:55:51.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:51 smithi135 ceph-mon[27511]: pgmap v644: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:55:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:55:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:55:53.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:53 smithi069 ceph-mon[25054]: pgmap v645: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:53.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:53 smithi135 ceph-mon[27511]: pgmap v645: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:55 smithi069 ceph-mon[25054]: pgmap v646: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:55 smithi135 ceph-mon[27511]: pgmap v646: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:56.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:56 smithi069 ceph-mon[25054]: pgmap v647: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:57.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:56 smithi135 ceph-mon[27511]: pgmap v647: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:55:59 smithi069 ceph-mon[25054]: pgmap v648: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:55:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:55:59 smithi135 ceph-mon[27511]: pgmap v648: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:01.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:01 smithi069 ceph-mon[25054]: pgmap v649: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:01.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:01 smithi135 ceph-mon[27511]: pgmap v649: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:03.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:03 smithi069 ceph-mon[25054]: pgmap v650: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:03.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:03 smithi135 ceph-mon[27511]: pgmap v650: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:05.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:05 smithi069 ceph-mon[25054]: pgmap v651: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:05.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:05 smithi135 ceph-mon[27511]: pgmap v651: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:06 smithi069 ceph-mon[25054]: pgmap v652: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:06 smithi135 ceph-mon[27511]: pgmap v652: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:07 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 2486.417ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:56:07.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:07 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 2486.568ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:56:08.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:07 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 2486.417ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:56:08.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:07 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 2486.568ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:56:08.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:08 smithi069 ceph-mon[25054]: pgmap v653: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:08 smithi135 ceph-mon[27511]: pgmap v653: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:11 smithi069 ceph-mon[25054]: pgmap v654: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:11.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:11 smithi135 ceph-mon[27511]: pgmap v654: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:13 smithi069 ceph-mon[25054]: pgmap v655: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:13.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:13 smithi135 ceph-mon[27511]: pgmap v655: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:15.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:15 smithi069 ceph-mon[25054]: pgmap v656: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:15.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:15 smithi135 ceph-mon[27511]: pgmap v656: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:16.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:16 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 1901.771ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:56:16.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:16 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 1828.251ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:56:16.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:16 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 1901.771ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:56:16.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:16 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 1828.251ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:56:17.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:17 smithi069 ceph-mon[25054]: pgmap v657: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:17.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:17 smithi135 ceph-mon[27511]: pgmap v657: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:19.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:19 smithi069 ceph-mon[25054]: pgmap v658: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:19.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:19 smithi135 ceph-mon[27511]: pgmap v658: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:21.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:21 smithi069 ceph-mon[25054]: pgmap v659: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:21 smithi135 ceph-mon[27511]: pgmap v659: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:23 smithi069 ceph-mon[25054]: pgmap v660: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:23.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:23 smithi135 ceph-mon[27511]: pgmap v660: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:25.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:25 smithi069 ceph-mon[25054]: pgmap v661: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:25.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:25 smithi135 ceph-mon[27511]: pgmap v661: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:26.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:26 smithi069 ceph-mon[25054]: pgmap v662: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:26 smithi135 ceph-mon[27511]: pgmap v662: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:29.437 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:29 smithi069 ceph-mon[25054]: pgmap v663: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:29.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:29 smithi135 ceph-mon[27511]: pgmap v663: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:31.039 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-06-11T13:56:31.392 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:31 smithi069 ceph-mon[25054]: pgmap v664: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:31.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:31 smithi135 ceph-mon[27511]: pgmap v664: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:33.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:33 smithi069 ceph-mon[25054]: pgmap v665: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:33.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:33 smithi135 ceph-mon[27511]: pgmap v665: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:35 smithi069 ceph-mon[25054]: pgmap v666: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:35.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:35 smithi135 ceph-mon[27511]: pgmap v666: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:36.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:36 smithi069 ceph-mon[25054]: pgmap v667: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:37.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:36 smithi135 ceph-mon[27511]: pgmap v667: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:37.588 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph config set global log_to_journald false --force' 2024-06-11T13:56:39.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:39 smithi069 ceph-mon[25054]: pgmap v668: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:39.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:39 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:39.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:39 smithi135 ceph-mon[27511]: pgmap v668: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:39.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:39 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:40.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:40 smithi069 ceph-mon[25054]: pgmap v669: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:40.703 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:40 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:40.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:40 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:40.704 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:40 smithi069 ceph-mon[25054]: Health check cleared: CEPHADM_REFRESH_FAILED (was: failed to probe daemons or devices) 2024-06-11T13:56:40.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:40 smithi135 ceph-mon[27511]: pgmap v669: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:40.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:40 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:40.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:40 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:40.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:40 smithi135 ceph-mon[27511]: Health check cleared: CEPHADM_REFRESH_FAILED (was: failed to probe daemons or devices) 2024-06-11T13:56:42.986 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:42 smithi069 ceph-mon[25054]: pgmap v670: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:42.989 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph mgr module enable nfs --force' 2024-06-11T13:56:43.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:42 smithi135 ceph-mon[27511]: pgmap v670: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:44.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:44 smithi069 ceph-mon[25054]: pgmap v671: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:45.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:44 smithi135 ceph-mon[27511]: pgmap v671: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:46.732 INFO:teuthology.orchestra.run.smithi069.stderr:module 'nfs' is already enabled 2024-06-11T13:56:46.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:46 smithi069 ceph-mon[25054]: pgmap v672: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:46.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:46 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/310817266' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-06-11T13:56:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:46 smithi135 ceph-mon[27511]: pgmap v672: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:47.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:46 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/310817266' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-06-11T13:56:47.744 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:47 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/310817266' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-06-11T13:56:47.744 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:47 smithi069 ceph-mon[25054]: mgrmap e21: smithi069.iruzyw(active, since 19m), standbys: smithi135.xcould 2024-06-11T13:56:48.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:47 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/310817266' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-06-11T13:56:48.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:47 smithi135 ceph-mon[27511]: mgrmap e21: smithi069.iruzyw(active, since 19m), standbys: smithi135.xcould 2024-06-11T13:56:48.102 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-06-11T13:56:48.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:48 smithi069 ceph-mon[25054]: pgmap v673: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:49.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:48 smithi135 ceph-mon[27511]: pgmap v673: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:51 smithi135 ceph-mon[27511]: pgmap v674: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:56:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:56:51.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:51 smithi069 ceph-mon[25054]: pgmap v674: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-06-11T13:56:51.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:51.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:56:51.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:56:51.394 INFO:teuthology.orchestra.run.smithi069.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d 2024-06-11T13:56:52.452 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-06-11T13:56:52.453 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T13:56:52.453 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-06-11T13:56:52.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:52.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d 2024-06-11T13:56:52.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:52.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:56:52.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:56:52.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:56:52.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: pgmap v675: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:52.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:52.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:52.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:52 smithi069 ceph-mon[25054]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d 2024-06-11T13:56:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d 2024-06-11T13:56:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:56:52.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:56:52.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:56:52.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: pgmap v675: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:56:52.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:52.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:56:52.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:52 smithi135 ceph-mon[27511]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d 2024-06-11T13:56:54.732 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:54 smithi069 ceph-mon[25054]: pgmap v676: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:54.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:54 smithi135 ceph-mon[27511]: pgmap v676: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:55.729 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T13:56:56.490 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:56:56.490 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (19m) 17s ago 20m 27.0M - ba2b418f427c 59bb0c633338 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (20m) 17s ago 20m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (19m) 2m ago 19m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (18m) 17s ago 20m 55.0M - 8.3.5 dad864ee21e9 c3e2fd2cea97 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (14m) 17s ago 14m 37.3M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (14m) 2m ago 14m 17.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:9283 running (21m) 17s ago 21m 469M - 17.2.0 e1d6a67b021e 15636a05991a 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283 running (19m) 2m ago 19m 418M - 17.2.0 e1d6a67b021e 7ad95b642fed 2024-06-11T13:56:56.491 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (21m) 17s ago 22m 55.8M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:56:56.492 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (19m) 2m ago 19m 42.6M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:56:56.492 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (14m) 17s ago 14m 244M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T13:56:56.492 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (14m) 2m ago 14m 86.2M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T13:56:56.492 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (20m) 17s ago 20m 17.6M - 1dbe0e931976 2b3eb7456a5b 2024-06-11T13:56:56.492 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (19m) 2m ago 19m 19.1M - 1dbe0e931976 3b824b2b7502 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (19m) 17s ago 19m 482M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (18m) 17s ago 18m 389M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (18m) 17s ago 18m 523M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (17m) 17s ago 17m 405M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (17m) 2m ago 17m 426M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (16m) 2m ago 16m 518M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:56:56.493 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (16m) 2m ago 16m 436M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:56:56.494 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (16m) 2m ago 16m 662M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:56:56.494 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (19m) 17s ago 19m 66.2M - 514e6a882f6e ccd8f2b5541e 2024-06-11T13:56:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:56 smithi069 ceph-mon[25054]: pgmap v677: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:56 smithi069 ceph-mon[25054]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:56 smithi069 ceph-mon[25054]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:56 smithi135 ceph-mon[27511]: pgmap v677: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:56 smithi135 ceph-mon[27511]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:56 smithi135 ceph-mon[27511]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:57.088 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:56:57.088 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T13:56:57.088 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:56:57.088 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T13:56:57.089 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:56:57.461 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:56:57.461 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d", 2024-06-11T13:56:57.461 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T13:56:57.462 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-06-11T13:56:57.462 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "", 2024-06-11T13:56:57.462 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3f51ea3944d39ca13aa40153c9a4ee298306679d image" 2024-06-11T13:56:57.462 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:56:57.872 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Slow OSD heartbeats on back (longest 1901.771ms); Slow OSD heartbeats on front (longest 1828.251ms) 2024-06-11T13:56:57.872 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1901.771ms) 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.1 [] to osd.0 [] 1901.771 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.4 [] to osd.0 [] 1705.138 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.5 [] to osd.1 [] 1569.828 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.1 [] 1517.520 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.1 [] 1439.129 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.3 [] to osd.1 [] 1424.920 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.0 [] 1375.771 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.0 [] 1350.525 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.1 [] 1310.486 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.0 [] 1244.810 msec 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Truncated long network list. Use ceph daemon mgr.# dump_osd_network for more information 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_FRONT: Slow OSD heartbeats on front (longest 1828.251ms) 2024-06-11T13:56:57.873 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.1 [] to osd.0 [] 1828.251 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.4 [] to osd.0 [] 1705.491 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.5 [] to osd.1 [] 1566.806 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.1 [] 1520.566 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.1 [] 1439.677 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.1 [] 1438.210 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.3 [] to osd.1 [] 1424.945 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.0 [] 1376.002 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.0 [] 1350.669 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.0 [] 1244.532 msec 2024-06-11T13:56:57.874 INFO:teuthology.orchestra.run.smithi069.stdout: Truncated long network list. Use ceph daemon mgr.# dump_osd_network for more information 2024-06-11T13:56:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:57 smithi069 ceph-mon[25054]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:57 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1742355118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:56:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:57 smithi135 ceph-mon[27511]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:58.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:57 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1742355118' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:56:58.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:58 smithi069 ceph-mon[25054]: pgmap v678: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:58.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:58 smithi069 ceph-mon[25054]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:58.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:56:58 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1336593677' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:56:59.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:58 smithi135 ceph-mon[27511]: pgmap v678: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:56:59.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:58 smithi135 ceph-mon[27511]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:56:59.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:56:58 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1336593677' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:57:00.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:00 smithi069 ceph-mon[25054]: pgmap v679: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:01.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:00 smithi135 ceph-mon[27511]: pgmap v679: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:02.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:02 smithi069 ceph-mon[25054]: pgmap v680: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:03.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:02 smithi135 ceph-mon[27511]: pgmap v680: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:04.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:04 smithi069 ceph-mon[25054]: pgmap v681: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:05.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:04 smithi135 ceph-mon[27511]: pgmap v681: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:06.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:06 smithi069 ceph-mon[25054]: pgmap v682: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:07.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:06 smithi135 ceph-mon[27511]: pgmap v682: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:08.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:08 smithi069 ceph-mon[25054]: pgmap v683: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:09.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:08 smithi135 ceph-mon[27511]: pgmap v683: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:10.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:09 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 1885.353ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:57:10.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:09 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 1811.716ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:57:10.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:09 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 1885.353ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:57:10.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:09 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 1811.716ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:57:11.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:10 smithi135 ceph-mon[27511]: pgmap v684: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:11.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:10 smithi069 ceph-mon[25054]: pgmap v684: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:13.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:12 smithi135 ceph-mon[27511]: pgmap v685: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:13.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:12 smithi069 ceph-mon[25054]: pgmap v685: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:14 smithi135 ceph-mon[27511]: pgmap v686: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:15.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:14 smithi069 ceph-mon[25054]: pgmap v686: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:17.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:16 smithi135 ceph-mon[27511]: pgmap v687: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:17.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:16 smithi069 ceph-mon[25054]: pgmap v687: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:18 smithi069 ceph-mon[25054]: pgmap v688: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:19.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:18 smithi135 ceph-mon[27511]: pgmap v688: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:21.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:20 smithi069 ceph-mon[25054]: pgmap v689: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:21.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:20 smithi135 ceph-mon[27511]: pgmap v689: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:22 smithi135 ceph-mon[27511]: pgmap v690: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:23.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:22 smithi069 ceph-mon[25054]: pgmap v690: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:25.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:24 smithi069 ceph-mon[25054]: pgmap v691: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:25.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:24 smithi135 ceph-mon[27511]: pgmap v691: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:26.949 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:26 smithi069 ceph-mon[25054]: pgmap v692: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:27.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:26 smithi135 ceph-mon[27511]: pgmap v692: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:28.225 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (19m) 50s ago 21m 27.0M - ba2b418f427c 59bb0c633338 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (21m) 50s ago 21m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (20m) 2m ago 20m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (19m) 50s ago 20m 55.0M - 8.3.5 dad864ee21e9 c3e2fd2cea97 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (15m) 50s ago 15m 37.3M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (15m) 2m ago 15m 17.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:9283 running (22m) 50s ago 22m 469M - 17.2.0 e1d6a67b021e 15636a05991a 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283 running (20m) 2m ago 20m 418M - 17.2.0 e1d6a67b021e 7ad95b642fed 2024-06-11T13:57:28.834 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (22m) 50s ago 22m 55.8M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (20m) 2m ago 20m 42.6M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (14m) 50s ago 14m 244M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (14m) 2m ago 14m 86.2M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (20m) 50s ago 20m 17.6M - 1dbe0e931976 2b3eb7456a5b 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (20m) 2m ago 20m 19.1M - 1dbe0e931976 3b824b2b7502 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (19m) 50s ago 19m 482M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (19m) 50s ago 19m 389M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (18m) 50s ago 18m 523M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (18m) 50s ago 18m 405M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (17m) 2m ago 17m 426M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (17m) 2m ago 17m 518M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (17m) 2m ago 17m 436M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (16m) 2m ago 16m 662M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:57:28.835 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (19m) 50s ago 19m 66.2M - 514e6a882f6e ccd8f2b5541e 2024-06-11T13:57:29.194 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:57:29.195 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:57:29.196 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T13:57:29.196 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-11T13:57:29.196 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T13:57:29.196 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:57:29.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: pgmap v693: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:57:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: Upgrade: Target is version 19.0.0-2976-g3f51ea39 (unknown) 2024-06-11T13:57:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310'] 2024-06-11T13:57:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:57:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:57:29.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:29 smithi135 ceph-mon[27511]: Upgrade: Need to upgrade myself (mgr.smithi069.iruzyw) 2024-06-11T13:57:29.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: pgmap v693: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:29.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:57:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: Upgrade: Target is version 19.0.0-2976-g3f51ea39 (unknown) 2024-06-11T13:57:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310'] 2024-06-11T13:57:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:57:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:57:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:29 smithi069 ceph-mon[25054]: Upgrade: Need to upgrade myself (mgr.smithi069.iruzyw) 2024-06-11T13:57:29.505 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:57:29.505 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T13:57:29.505 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T13:57:29.505 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-06-11T13:57:29.505 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "0/23 daemons upgraded", 2024-06-11T13:57:29.506 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 image on host smithi135" 2024-06-11T13:57:29.506 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Slow OSD heartbeats on back (longest 1885.353ms); Slow OSD heartbeats on front (longest 1811.716ms) 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1885.353ms) 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.1 [] to osd.0 [] 1885.353 msec 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.4 [] to osd.0 [] 1705.138 msec 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.5 [] to osd.1 [] 1566.283 msec 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.1 [] 1515.040 msec 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.1 [] 1437.006 msec 2024-06-11T13:57:29.869 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.3 [] to osd.1 [] 1423.192 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.0 [] 1373.073 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.0 [] 1343.797 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.1 [] 1308.830 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.0 [] 1244.822 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Truncated long network list. Use ceph daemon mgr.# dump_osd_network for more information 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_FRONT: Slow OSD heartbeats on front (longest 1811.716ms) 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.1 [] to osd.0 [] 1811.716 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.4 [] to osd.0 [] 1705.491 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.5 [] to osd.1 [] 1565.740 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.1 [] 1515.515 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.1 [] 1437.908 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.1 [] 1436.541 msec 2024-06-11T13:57:29.870 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.3 [] to osd.1 [] 1423.219 msec 2024-06-11T13:57:29.871 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.0 [] 1373.305 msec 2024-06-11T13:57:29.871 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.0 [] 1343.977 msec 2024-06-11T13:57:29.871 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.0 [] 1244.545 msec 2024-06-11T13:57:29.871 INFO:teuthology.orchestra.run.smithi069.stdout: Truncated long network list. Use ceph daemon mgr.# dump_osd_network for more information 2024-06-11T13:57:30.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:30 smithi069 ceph-mon[25054]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:30.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:30 smithi069 ceph-mon[25054]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 on smithi135 2024-06-11T13:57:30.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:30 smithi069 ceph-mon[25054]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:30.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:30 smithi069 ceph-mon[25054]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:30.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:30 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/930462556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:57:30.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:30 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/1417740906' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:57:30.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:30 smithi135 ceph-mon[27511]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:30.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:30 smithi135 ceph-mon[27511]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 on smithi135 2024-06-11T13:57:30.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:30 smithi135 ceph-mon[27511]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:30.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:30 smithi135 ceph-mon[27511]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:30.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:30 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/930462556' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:57:30.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:30 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/1417740906' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:57:31.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:31 smithi135 ceph-mon[27511]: pgmap v694: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:31.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:31 smithi135 ceph-mon[27511]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:31.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:31 smithi069 ceph-mon[25054]: pgmap v694: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:31.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:31 smithi069 ceph-mon[25054]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:57:33.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:33 smithi135 ceph-mon[27511]: pgmap v695: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:33.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:33 smithi069 ceph-mon[25054]: pgmap v695: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:35.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:35 smithi135 ceph-mon[27511]: pgmap v696: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:35 smithi069 ceph-mon[25054]: pgmap v696: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:37.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:37 smithi135 ceph-mon[27511]: pgmap v697: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:37.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:37 smithi069 ceph-mon[25054]: pgmap v697: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:39 smithi135 ceph-mon[27511]: pgmap v698: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:39.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:39 smithi069 ceph-mon[25054]: pgmap v698: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:41.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:41 smithi135 ceph-mon[27511]: pgmap v699: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:41 smithi069 ceph-mon[25054]: pgmap v699: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:43.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:43 smithi135 ceph-mon[27511]: pgmap v700: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:43.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:43 smithi069 ceph-mon[25054]: pgmap v700: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:45.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:45 smithi135 ceph-mon[27511]: pgmap v701: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:45.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:45 smithi069 ceph-mon[25054]: pgmap v701: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:47.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:47 smithi135 ceph-mon[27511]: pgmap v702: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:47.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:47 smithi069 ceph-mon[25054]: pgmap v702: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:49.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:49 smithi135 ceph-mon[27511]: pgmap v703: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:49.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:49 smithi069 ceph-mon[25054]: pgmap v703: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:51 smithi135 ceph-mon[27511]: pgmap v704: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:57:51.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:51 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:57:51.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:51 smithi069 ceph-mon[25054]: pgmap v704: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:57:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:51 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:57:53.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:53 smithi135 ceph-mon[27511]: pgmap v705: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:53.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:53 smithi069 ceph-mon[25054]: pgmap v705: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:57:55.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:55 smithi135 ceph-mon[27511]: pgmap v706: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:55 smithi069 ceph-mon[25054]: pgmap v706: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:57.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:57 smithi135 ceph-mon[27511]: pgmap v707: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:57.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:57 smithi069 ceph-mon[25054]: pgmap v707: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:59.116 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:59 smithi135 ceph-mon[27511]: pgmap v708: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:59.116 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:57:59.116 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:57:59.116 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:57:59.117 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:57:59 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:57:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:59 smithi069 ceph-mon[25054]: pgmap v708: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:57:59.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:57:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:57:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:57:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:57:59 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:00.201 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T13:58:00.244 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:00 smithi135 ceph-mon[27511]: Upgrade: Updating mgr.smithi135.xcould 2024-06-11T13:58:00.245 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:00 smithi135 ceph-mon[27511]: Deploying daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:58:00.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:00 smithi069 ceph-mon[25054]: Upgrade: Updating mgr.smithi135.xcould 2024-06-11T13:58:00.312 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:00 smithi069 ceph-mon[25054]: Deploying daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:58:00.799 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:58:00.799 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (20m) 82s ago 21m 27.0M - ba2b418f427c 59bb0c633338 2024-06-11T13:58:00.799 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (21m) 82s ago 21m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (20m) 3m ago 20m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (19m) 82s ago 21m 55.0M - 8.3.5 dad864ee21e9 c3e2fd2cea97 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (15m) 82s ago 15m 37.3M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (15m) 3m ago 15m 17.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:9283 running (22m) 82s ago 22m 469M - 17.2.0 e1d6a67b021e 15636a05991a 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283 running (20m) 3m ago 20m 418M - 17.2.0 e1d6a67b021e 7ad95b642fed 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (23m) 82s ago 23m 55.8M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (20m) 3m ago 20m 42.6M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (15m) 82s ago 15m 244M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (15m) 3m ago 15m 86.2M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (21m) 82s ago 21m 17.6M - 1dbe0e931976 2b3eb7456a5b 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (20m) 3m ago 20m 19.1M - 1dbe0e931976 3b824b2b7502 2024-06-11T13:58:00.800 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (20m) 82s ago 20m 482M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (19m) 82s ago 19m 389M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (19m) 82s ago 19m 523M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (18m) 82s ago 18m 405M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (18m) 3m ago 18m 426M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (18m) 3m ago 18m 518M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (17m) 3m ago 17m 436M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (17m) 3m ago 17m 662M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:58:00.801 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (20m) 82s ago 20m 66.2M - 514e6a882f6e ccd8f2b5541e 2024-06-11T13:58:01.156 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:58:01.157 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:01.158 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T13:58:01.158 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-06-11T13:58:01.158 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T13:58:01.158 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:58:01.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:01 smithi135 ceph-mon[27511]: pgmap v709: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:01.450 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:58:01.450 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T13:58:01.451 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T13:58:01.451 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-06-11T13:58:01.451 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "0/23 daemons upgraded", 2024-06-11T13:58:01.451 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mgr daemons" 2024-06-11T13:58:01.451 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:58:01.451 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:01 smithi069 ceph-mon[25054]: pgmap v709: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:01.806 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Slow OSD heartbeats on back (longest 1885.353ms); Slow OSD heartbeats on front (longest 1811.716ms) 2024-06-11T13:58:01.806 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1885.353ms) 2024-06-11T13:58:01.806 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.1 [] to osd.0 [] 1885.353 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.4 [] to osd.0 [] 1705.134 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.5 [] to osd.1 [] 1566.283 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.1 [] 1515.040 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.1 [] 1437.006 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.3 [] to osd.1 [] 1423.192 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.0 [] 1373.073 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.0 [] 1343.797 msec 2024-06-11T13:58:01.807 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.1 [] 1308.830 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.0 [] 1244.822 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Truncated long network list. Use ceph daemon mgr.# dump_osd_network for more information 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_FRONT: Slow OSD heartbeats on front (longest 1811.716ms) 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.1 [] to osd.0 [] 1811.716 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.4 [] to osd.0 [] 1705.491 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.5 [] to osd.1 [] 1565.740 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.1 [] 1515.515 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.1 [] 1437.908 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.1 [] 1436.541 msec 2024-06-11T13:58:01.808 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.3 [] to osd.1 [] 1423.219 msec 2024-06-11T13:58:01.809 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.0 [] 1373.305 msec 2024-06-11T13:58:01.809 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.0 [] 1343.977 msec 2024-06-11T13:58:01.809 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.0 [] 1244.545 msec 2024-06-11T13:58:01.809 INFO:teuthology.orchestra.run.smithi069.stdout: Truncated long network list. Use ceph daemon mgr.# dump_osd_network for more information 2024-06-11T13:58:02.224 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2974910243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:58:02.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:02 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2038300148' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:58:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:02.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2974910243' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:58:02.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:02.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:02.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:02.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:58:02.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:02 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2038300148' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:58:03.292 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:03 smithi135 ceph-mon[27511]: pgmap v710: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:58:03.292 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:03 smithi135 ceph-mon[27511]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:03.433 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:03 smithi069 ceph-mon[25054]: pgmap v710: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:58:03.433 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:03 smithi069 ceph-mon[25054]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:05.091 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:05 smithi135 ceph-mon[27511]: pgmap v711: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:05.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:05 smithi069 ceph-mon[25054]: pgmap v711: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:06.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:06.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:06 smithi135 ceph-mon[27511]: pgmap v712: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:06.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:06 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:06.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:06.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:06 smithi069 ceph-mon[25054]: pgmap v712: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:06.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:06 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:07.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:07 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:58:07.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:07 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould started 2024-06-11T13:58:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:07 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:58:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:07 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:07 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:58:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:07 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:07.540 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:07 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:58:07.540 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:07 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould started 2024-06-11T13:58:07.540 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:07 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:58:07.540 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:07 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:07.540 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:07 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:58:07.540 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:07 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:08.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:08 smithi069 ceph-mon[25054]: mgrmap e22: smithi069.iruzyw(active, since 21m), standbys: smithi135.xcould 2024-06-11T13:58:08.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:08 smithi069 ceph-mon[25054]: pgmap v713: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:08.504 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:08 smithi135 ceph-mon[27511]: mgrmap e22: smithi069.iruzyw(active, since 21m), standbys: smithi135.xcould 2024-06-11T13:58:08.504 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:08 smithi135 ceph-mon[27511]: pgmap v713: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:09.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:09 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 1705.134ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:58:09.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:09 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 1705.491ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:58:09.773 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:09 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 1705.134ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:58:09.773 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:09 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 1705.491ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:58:11.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: pgmap v714: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:11.126 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: Upgrade: Need to upgrade myself (mgr.smithi069.iruzyw) 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: Failing over to other MGR 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr fail", "who": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: osdmap e82: 8 total, 8 up, 8 in 2024-06-11T13:58:11.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:10 smithi069 ceph-mon[25054]: Standby manager daemon smithi069.iruzyw started 2024-06-11T13:58:11.173 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: pgmap v714: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:11.173 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:11.173 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: Upgrade: Need to upgrade myself (mgr.smithi069.iruzyw) 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: Failing over to other MGR 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr fail", "who": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: osdmap e82: 8 total, 8 up, 8 in 2024-06-11T13:58:11.174 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:10 smithi135 ceph-mon[27511]: Standby manager daemon smithi069.iruzyw started 2024-06-11T13:58:12.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "mgr fail", "who": "smithi069.iruzyw"}]': finished 2024-06-11T13:58:12.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: mgrmap e23: smithi135.xcould(active, starting, since 1.00314s), standbys: smithi069.iruzyw 2024-06-11T13:58:12.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:58:12.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:58:12.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:58:12.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:58:12.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:12.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:58:12.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:58:12.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:58:12.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:58:12.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:58:12.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:58:12.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: Manager daemon smithi135.xcould is now available 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/trash_purge_schedule"}]: dispatch 2024-06-11T13:58:12.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/trash_purge_schedule"}]: dispatch 2024-06-11T13:58:12.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:12.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:11 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.14204 172.21.15.69:0/1347003055' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "mgr fail", "who": "smithi069.iruzyw"}]': finished 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: mgrmap e23: smithi135.xcould(active, starting, since 1.00314s), standbys: smithi069.iruzyw 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:58:12.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: Manager daemon smithi135.xcould is now available 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:12.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:58:12.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:58:12.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/trash_purge_schedule"}]: dispatch 2024-06-11T13:58:12.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi135.xcould/trash_purge_schedule"}]: dispatch 2024-06-11T13:58:12.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:12.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:11 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: Deploying cephadm binary to smithi069 2024-06-11T13:58:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: Deploying cephadm binary to smithi135 2024-06-11T13:58:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: mgrmap e24: smithi135.xcould(active, since 2s), standbys: smithi069.iruzyw 2024-06-11T13:58:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: pgmap v3: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:12] ENGINE Bus STARTING 2024-06-11T13:58:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:12] ENGINE Serving on https://172.21.15.135:7150 2024-06-11T13:58:12.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:12 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:12] ENGINE Error in HTTPServer.serve 2024-06-11T13:58:12.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Traceback (most recent call last): 2024-06-11T13:58:12.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-11T13:58:12.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._connections.run(self.expiration_interval) 2024-06-11T13:58:12.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._run(expiration_interval) 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-11T13:58:13.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: s = self.context.wrap_socket( 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: return self.sslsocket_class._create( 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self.do_handshake() 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._sslobj.do_handshake() 2024-06-11T13:58:13.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-11T13:58:13.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: Deploying cephadm binary to smithi069 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: Deploying cephadm binary to smithi135 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: mgrmap e24: smithi135.xcould(active, since 2s), standbys: smithi069.iruzyw 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: pgmap v3: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:12] ENGINE Bus STARTING 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:12] ENGINE Serving on https://172.21.15.135:7150 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:12 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:12] ENGINE Error in HTTPServer.serve 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Traceback (most recent call last): 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._connections.run(self.expiration_interval) 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-11T13:58:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._run(expiration_interval) 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: s = self.context.wrap_socket( 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: return self.sslsocket_class._create( 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self.do_handshake() 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-11T13:58:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._sslobj.do_handshake() 2024-06-11T13:58:13.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-11T13:58:14.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:13 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:12] ENGINE Serving on http://172.21.15.135:8765 2024-06-11T13:58:14.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:13 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:12] ENGINE Bus STARTED 2024-06-11T13:58:14.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:13 smithi135 ceph-mon[27511]: pgmap v4: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:14.136 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:13 smithi135 ceph-mon[27511]: mgrmap e25: smithi135.xcould(active, since 3s), standbys: smithi069.iruzyw 2024-06-11T13:58:14.136 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:13 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:14.136 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:13 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:13 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:12] ENGINE Serving on http://172.21.15.135:8765 2024-06-11T13:58:14.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:13 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:12] ENGINE Bus STARTED 2024-06-11T13:58:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:13 smithi069 ceph-mon[25054]: pgmap v4: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:13 smithi069 ceph-mon[25054]: mgrmap e25: smithi135.xcould(active, since 3s), standbys: smithi069.iruzyw 2024-06-11T13:58:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:13 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:14.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:13 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:16.110 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:15 smithi069 ceph-mon[25054]: pgmap v5: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:16.110 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:15 smithi069 ceph-mon[25054]: mgrmap e26: smithi135.xcould(active, since 5s), standbys: smithi069.iruzyw 2024-06-11T13:58:16.270 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:15 smithi135 ceph-mon[27511]: pgmap v5: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:16.270 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:15 smithi135 ceph-mon[27511]: mgrmap e26: smithi135.xcould(active, since 5s), standbys: smithi069.iruzyw 2024-06-11T13:58:17.986 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.986 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.986 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: pgmap v6: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:17.986 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: Standby manager daemon smithi069.iruzyw restarted 2024-06-11T13:58:17.986 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: Standby manager daemon smithi069.iruzyw started 2024-06-11T13:58:17.987 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/crt"}]: dispatch 2024-06-11T13:58:17.987 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:17.987 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/key"}]: dispatch 2024-06-11T13:58:17.987 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:17.987 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.987 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:17 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: pgmap v6: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:17.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: Standby manager daemon smithi069.iruzyw restarted 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: Standby manager daemon smithi069.iruzyw started 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/crt"}]: dispatch 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/key"}]: dispatch 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1515976208' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:17 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:18 smithi069 ceph-mon[25054]: mgrmap e27: smithi135.xcould(active, since 7s), standbys: smithi069.iruzyw 2024-06-11T13:58:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:18 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:18.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:18 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:18.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:18 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:18.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:18 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:19.025 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:18 smithi135 ceph-mon[27511]: mgrmap e27: smithi135.xcould(active, since 7s), standbys: smithi069.iruzyw 2024-06-11T13:58:19.025 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:18 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:19.025 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:18 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:19.025 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:18 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:19.025 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:18 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:20.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:19 smithi135 ceph-mon[27511]: pgmap v7: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-11T13:58:20.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:19 smithi069 ceph-mon[25054]: pgmap v7: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.928 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:21.929 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:21 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:21 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:22.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:58:22.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: pgmap v8: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-06-11T13:58:22.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:22 smithi069 ceph-mon[25054]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: pgmap v8: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Reconfiguring alertmanager.smithi069 (dependencies changed)... 2024-06-11T13:58:23.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:22 smithi135 ceph-mon[27511]: Reconfiguring daemon alertmanager.smithi069 on smithi069 2024-06-11T13:58:24.740 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:24 smithi069 ceph-mon[25054]: pgmap v9: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-11T13:58:25.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:24 smithi135 ceph-mon[27511]: pgmap v9: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: pgmap v10: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:26.990 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:26 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:58:27.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: pgmap v10: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:58:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:27.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:26 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:58:27.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:27 smithi069 ceph-mon[25054]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-06-11T13:58:27.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:27 smithi069 ceph-mon[25054]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-11T13:58:27.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:27 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:58:27.967 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:27 smithi069 ceph-mon[25054]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-06-11T13:58:28.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:27 smithi135 ceph-mon[27511]: Reconfiguring grafana.smithi069 (dependencies changed)... 2024-06-11T13:58:28.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:27 smithi135 ceph-mon[27511]: Regenerating cephadm self-signed grafana TLS certificates 2024-06-11T13:58:28.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:27 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-06-11T13:58:28.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:27 smithi135 ceph-mon[27511]: Reconfiguring daemon grafana.smithi069 on smithi069 2024-06-11T13:58:28.918 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:28 smithi069 ceph-mon[25054]: pgmap v11: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:58:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:28 smithi135 ceph-mon[27511]: pgmap v11: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:58:30.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:30 smithi069 ceph-mon[25054]: pgmap v12: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:58:30.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:30 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:30.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:30 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:30 smithi135 ceph-mon[27511]: pgmap v12: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:58:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:30 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:31.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:30 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:31.614 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:31 smithi069 ceph-mon[25054]: Reconfiguring node-exporter.smithi069 (dependencies changed)... 2024-06-11T13:58:31.615 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:31 smithi069 ceph-mon[25054]: Reconfiguring daemon node-exporter.smithi069 on smithi069 2024-06-11T13:58:32.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:31 smithi135 ceph-mon[27511]: Reconfiguring node-exporter.smithi069 (dependencies changed)... 2024-06-11T13:58:32.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:31 smithi135 ceph-mon[27511]: Reconfiguring daemon node-exporter.smithi069 on smithi069 2024-06-11T13:58:32.159 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T13:58:32.847 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 starting - - - - 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (22m) 14s ago 22m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (21m) 16s ago 21m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 starting - - - - 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (16m) 14s ago 16m 39.2M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (16m) 16s ago 16m 18.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:9283 running (23m) 14s ago 23m 207M - 17.2.0 e1d6a67b021e 15636a05991a 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283 running (31s) 16s ago 21m 483M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 923ca2e08e50 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (23m) 14s ago 23m 66.1M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (21m) 16s ago 21m 47.2M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (15m) 14s ago 15m 252M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T13:58:32.848 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (15m) 16s ago 15m 87.0M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 starting - - - - 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (21m) 16s ago 21m 21.2M - 1.3.1 1dbe0e931976 3b824b2b7502 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (20m) 14s ago 20m 499M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (20m) 14s ago 20m 409M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (19m) 14s ago 19m 527M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (19m) 14s ago 19m 413M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (18m) 16s ago 18m 458M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (18m) 16s ago 18m 533M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (18m) 16s ago 18m 454M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (17m) 16s ago 17m 686M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:58:32.849 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (20m) 14s ago 20m 68.3M - 2.33.4 514e6a882f6e ccd8f2b5541e 2024-06-11T13:58:32.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:32 smithi069 ceph-mon[25054]: pgmap v13: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:32.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:32 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:32.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:32 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:32 smithi135 ceph-mon[27511]: pgmap v13: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:32 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:32 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:33.249 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:58:33.249 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 1 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-06-11T13:58:33.250 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 1 2024-06-11T13:58:33.251 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T13:58:33.251 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:58:33.591 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:58:33.591 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T13:58:33.591 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T13:58:33.591 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T13:58:33.592 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-06-11T13:58:33.592 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "1/23 daemons upgraded", 2024-06-11T13:58:33.592 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-06-11T13:58:33.592 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T13:58:33.592 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:58:33.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:33 smithi069 ceph-mon[25054]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-06-11T13:58:33.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:33 smithi069 ceph-mon[25054]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:33.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:33 smithi069 ceph-mon[25054]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-06-11T13:58:33.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:33 smithi069 ceph-mon[25054]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:33.886 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:33 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3587748147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:58:33.997 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Slow OSD heartbeats on back (longest 1705.134ms); Slow OSD heartbeats on front (longest 1705.491ms) 2024-06-11T13:58:33.997 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1705.134ms) 2024-06-11T13:58:33.997 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.4 [] to osd.0 [] 1705.134 msec 2024-06-11T13:58:33.997 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.5 [] to osd.1 [] 1566.291 msec 2024-06-11T13:58:33.997 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.1 [] 1509.874 msec 2024-06-11T13:58:33.998 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.7 [] to osd.1 [] 1437.015 msec 2024-06-11T13:58:33.998 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.3 [] to osd.1 [] 1422.180 msec 2024-06-11T13:58:33.998 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.6 [] to osd.1 [] 1308.832 msec 2024-06-11T13:58:33.998 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.4 [] to osd.1 [] 1239.556 msec 2024-06-11T13:58:33.998 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.1 [] to osd.0 [] 1194.042 msec 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.0 [] to osd.1 [] 1028.935 msec 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_FRONT: Slow OSD heartbeats on front (longest 1705.491ms) 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.4 [] to osd.0 [] 1705.491 msec 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.5 [] to osd.1 [] 1565.748 msec 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.1 [] 1510.353 msec 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.7 [] to osd.1 [] 1437.912 msec 2024-06-11T13:58:33.999 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.6 [] to osd.1 [] 1436.546 msec 2024-06-11T13:58:34.000 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.3 [] to osd.1 [] 1422.203 msec 2024-06-11T13:58:34.000 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.4 [] to osd.1 [] 1239.094 msec 2024-06-11T13:58:34.000 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.1 [] to osd.0 [] 1120.440 msec 2024-06-11T13:58:34.000 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.0 [] to osd.1 [] 1028.932 msec 2024-06-11T13:58:34.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:33 smithi135 ceph-mon[27511]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-06-11T13:58:34.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:33 smithi135 ceph-mon[27511]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:34.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:33 smithi135 ceph-mon[27511]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-06-11T13:58:34.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:33 smithi135 ceph-mon[27511]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:34.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:33 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3587748147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:58:34.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:34 smithi069 ceph-mon[25054]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:34.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:34 smithi069 ceph-mon[25054]: pgmap v14: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:58:34.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:34 smithi069 ceph-mon[25054]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:34.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:34 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/2980439442' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:58:35.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:34 smithi135 ceph-mon[27511]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:34 smithi135 ceph-mon[27511]: pgmap v14: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:58:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:34 smithi135 ceph-mon[27511]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:58:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:34 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2980439442' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:58:36.948 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:36 smithi135 ceph-mon[27511]: pgmap v15: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:36.948 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:36 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:36.948 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:36 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:36.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:36 smithi069 ceph-mon[25054]: pgmap v15: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:36.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:36 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:36.968 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:36 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:37.900 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:37 smithi135 ceph-mon[27511]: Reconfiguring node-exporter.smithi135 (dependencies changed)... 2024-06-11T13:58:37.900 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:37 smithi135 ceph-mon[27511]: Reconfiguring daemon node-exporter.smithi135 on smithi135 2024-06-11T13:58:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:37 smithi069 ceph-mon[25054]: Reconfiguring node-exporter.smithi135 (dependencies changed)... 2024-06-11T13:58:37.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:37 smithi069 ceph-mon[25054]: Reconfiguring daemon node-exporter.smithi135 on smithi135 2024-06-11T13:58:38.733 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: pgmap v16: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:38.733 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on back (longest 1566.291ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: Health check update: Slow OSD heartbeats on front (longest 1565.748ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:58:38.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-06-11T13:58:38.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:38 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:38.954 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: pgmap v16: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on back (longest 1566.291ms) (OSD_SLOW_PING_TIME_BACK) 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: Health check update: Slow OSD heartbeats on front (longest 1565.748ms) (OSD_SLOW_PING_TIME_FRONT) 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:58:38.955 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-06-11T13:58:38.956 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:38.956 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:38 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:39.879 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:39.880 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:39.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:58:39.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:58:39.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:58:39.881 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:39 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:40.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T13:58:40.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi069.front.sepia.ceph.com:9093"}]: dispatch 2024-06-11T13:58:40.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T13:58:40.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi069.front.sepia.ceph.com:3000"}]: dispatch 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi069.front.sepia.ceph.com:9095"}]: dispatch 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:58:40.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:58:40.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:58:40.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:39 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:40.812 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:40 smithi069 ceph-mon[25054]: pgmap v17: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:40.813 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:40 smithi069 ceph-mon[25054]: Upgrade: Updating mgr.smithi069.iruzyw 2024-06-11T13:58:40.813 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:40 smithi069 ceph-mon[25054]: Deploying daemon mgr.smithi069.iruzyw on smithi069 2024-06-11T13:58:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:40 smithi135 ceph-mon[27511]: pgmap v17: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:40 smithi135 ceph-mon[27511]: Upgrade: Updating mgr.smithi069.iruzyw 2024-06-11T13:58:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:40 smithi135 ceph-mon[27511]: Deploying daemon mgr.smithi069.iruzyw on smithi069 2024-06-11T13:58:41.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:41 smithi069 ceph-mon[25054]: pgmap v18: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:41.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:41 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:41 smithi135 ceph-mon[27511]: pgmap v18: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:42.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:41 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:44.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:43 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:44.076 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:43 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:44.076 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:43 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:44.076 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:43 smithi135 ceph-mon[27511]: pgmap v19: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:58:44.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:43 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:44.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:43 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:44.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:43 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:44.191 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:43 smithi069 ceph-mon[25054]: pgmap v19: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:58:45.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:45 smithi069 ceph-mon[25054]: pgmap v20: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:46.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:45 smithi135 ceph-mon[27511]: pgmap v20: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:47.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:47 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:47 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:47.587 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:47 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:47.587 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:47 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: pgmap v21: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: Standby manager daemon smithi069.iruzyw restarted 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: Standby manager daemon smithi069.iruzyw started 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/crt"}]: dispatch 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/key"}]: dispatch 2024-06-11T13:58:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:48 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: pgmap v21: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: Standby manager daemon smithi069.iruzyw restarted 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: Standby manager daemon smithi069.iruzyw started 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/crt"}]: dispatch 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi069.iruzyw/key"}]: dispatch 2024-06-11T13:58:48.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:48 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:49.618 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:49 smithi069 ceph-mon[25054]: mgrmap e28: smithi135.xcould(active, since 38s), standbys: smithi069.iruzyw 2024-06-11T13:58:49.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:49 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:49 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:49 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.619 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:49 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:49 smithi135 ceph-mon[27511]: mgrmap e28: smithi135.xcould(active, since 38s), standbys: smithi069.iruzyw 2024-06-11T13:58:49.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:49 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:49 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:49 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:49.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:49 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:50.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:50 smithi069 ceph-mon[25054]: pgmap v22: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:50.796 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:50 smithi135 ceph-mon[27511]: pgmap v22: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:51.980 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:51 smithi069 ceph-mon[25054]: pgmap v23: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:52.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:51 smithi135 ceph-mon[27511]: pgmap v23: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:58:53.477 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: Failing over to other MGR 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr fail", "who": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr fail", "who": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:53.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: osdmap e83: 8 total, 8 up, 8 in 2024-06-11T13:58:53.479 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:53 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould started 2024-06-11T13:58:53.571 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.571 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: Upgrade: Need to upgrade myself (mgr.smithi135.xcould) 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: Failing over to other MGR 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 172.21.15.135:0/1798609896' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr fail", "who": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd=[{"prefix": "mgr fail", "who": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: osdmap e83: 8 total, 8 up, 8 in 2024-06-11T13:58:53.572 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:53 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould started 2024-06-11T13:58:54.658 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd='[{"prefix": "mgr fail", "who": "smithi135.xcould"}]': finished 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: mgrmap e29: smithi069.iruzyw(active, starting, since 0.884504s), standbys: smithi135.xcould 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:58:54.659 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:58:54.660 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:54 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.24425 ' entity='mgr.smithi135.xcould' cmd='[{"prefix": "mgr fail", "who": "smithi135.xcould"}]': finished 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: mgrmap e29: smithi069.iruzyw(active, starting, since 0.884504s), standbys: smithi135.xcould 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:58:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:58:54.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:54 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:58:55.605 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:55 smithi069 ceph-mon[25054]: mgrmap e30: smithi069.iruzyw(active, since 2s), standbys: smithi135.xcould 2024-06-11T13:58:55.605 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:55 smithi069 ceph-mon[25054]: pgmap v3: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:55.748 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:55 smithi135 ceph-mon[27511]: mgrmap e30: smithi069.iruzyw(active, since 2s), standbys: smithi135.xcould 2024-06-11T13:58:55.748 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:55 smithi135 ceph-mon[27511]: pgmap v3: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:56 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:54] ENGINE Bus STARTING 2024-06-11T13:58:56.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:56 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:55] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:56 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:55] ENGINE Error in HTTPServer.serve 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Traceback (most recent call last): 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._connections.run(self.expiration_interval) 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._run(expiration_interval) 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: s = self.context.wrap_socket( 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-11T13:58:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: return self.sslsocket_class._create( 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self.do_handshake() 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._sslobj.do_handshake() 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:56 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:55] ENGINE Serving on http://172.21.15.69:8765 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:56 smithi069 ceph-mon[25054]: [11/Jun/2024:13:58:55] ENGINE Bus STARTED 2024-06-11T13:58:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:56 smithi069 ceph-mon[25054]: pgmap v4: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:56 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:54] ENGINE Bus STARTING 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:56 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:55] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:56 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:55] ENGINE Error in HTTPServer.serve 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Traceback (most recent call last): 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._connections.run(self.expiration_interval) 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._run(expiration_interval) 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-11T13:58:56.786 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: s = self.context.wrap_socket( 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: return self.sslsocket_class._create( 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self.do_handshake() 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._sslobj.do_handshake() 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:56 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:55] ENGINE Serving on http://172.21.15.69:8765 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:56 smithi135 ceph-mon[27511]: [11/Jun/2024:13:58:55] ENGINE Bus STARTED 2024-06-11T13:58:56.787 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:56 smithi135 ceph-mon[27511]: pgmap v4: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:57 smithi069 ceph-mon[25054]: mgrmap e31: smithi069.iruzyw(active, since 4s), standbys: smithi135.xcould 2024-06-11T13:58:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:57 smithi069 ceph-mon[25054]: pgmap v5: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:57 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:57 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:58.046 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:57 smithi135 ceph-mon[27511]: mgrmap e31: smithi069.iruzyw(active, since 4s), standbys: smithi135.xcould 2024-06-11T13:58:58.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:57 smithi135 ceph-mon[27511]: pgmap v5: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:58:58.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:57 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:58.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:57 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:59.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould started 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:58:59.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:58 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould started 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:58:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:58 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/4167424830' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:59:00.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:59 smithi069 ceph-mon[25054]: mgrmap e32: smithi069.iruzyw(active, since 6s), standbys: smithi135.xcould 2024-06-11T13:59:00.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:59 smithi069 ceph-mon[25054]: pgmap v6: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:00.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:59 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:00.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:58:59 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:59 smithi135 ceph-mon[27511]: mgrmap e32: smithi069.iruzyw(active, since 6s), standbys: smithi135.xcould 2024-06-11T13:59:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:59 smithi135 ceph-mon[27511]: pgmap v6: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:59 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:58:59 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:01.430 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:01 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:01.430 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:01 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:01 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:01 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:02.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:02 smithi069 ceph-mon[25054]: pgmap v7: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-11T13:59:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:02 smithi135 ceph-mon[27511]: pgmap v7: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-11T13:59:04.360 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T13:59:04.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: pgmap v8: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:04 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: pgmap v8: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:04 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (39s) 5s ago 22m 16.4M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (22m) 5s ago 22m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (22m) 7s ago 22m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (35s) 5s ago 22m 43.0M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (16m) 5s ago 16m 39.3M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (16m) 7s ago 16m 18.3M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (22s) 5s ago 24m 474M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283 running (63s) 7s ago 21m 82.7M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 923ca2e08e50 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (24m) 5s ago 24m 69.7M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:59:05.026 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (21m) 7s ago 21m 50.9M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (16m) 5s ago 16m 268M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (16m) 7s ago 16m 87.0M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (33s) 5s ago 22m 8224k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (27s) 7s ago 21m 9903k - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (21m) 5s ago 21m 499M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (20m) 5s ago 20m 411M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (20m) 5s ago 20m 527M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (19m) 5s ago 19m 414M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (19m) 7s ago 19m 458M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (19m) 7s ago 19m 534M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (18m) 7s ago 18m 455M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (18m) 7s ago 18m 688M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:59:05.027 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (28s) 5s ago 21m 53.6M - 2.33.4 514e6a882f6e 5dd91be11600 2024-06-11T13:59:05.443 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:59:05.443 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T13:59:05.443 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:59:05.443 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:05.443 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T13:59:05.443 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T13:59:05.444 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:59:05.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:05 smithi069 ceph-mon[25054]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:05.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:05 smithi069 ceph-mon[25054]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:05.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:05 smithi069 ceph-mon[25054]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-06-11T13:59:05.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:05 smithi069 ceph-mon[25054]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:05.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:05 smithi069 ceph-mon[25054]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-06-11T13:59:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:05 smithi135 ceph-mon[27511]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:05 smithi135 ceph-mon[27511]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:05 smithi135 ceph-mon[27511]: Reconfiguring prometheus.smithi069 (dependencies changed)... 2024-06-11T13:59:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:05 smithi135 ceph-mon[27511]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:05.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:05 smithi135 ceph-mon[27511]: Reconfiguring daemon prometheus.smithi069 on smithi069 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "2/23 daemons upgraded", 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-06-11T13:59:05.783 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T13:59:05.784 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:59:06.183 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Slow OSD heartbeats on back (longest 1566.291ms); Slow OSD heartbeats on front (longest 1565.748ms) 2024-06-11T13:59:06.183 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_BACK: Slow OSD heartbeats on back (longest 1566.291ms) 2024-06-11T13:59:06.183 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.5 [] to osd.1 [] 1566.291 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.2 [] to osd.1 [] 1509.874 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.1 [] to osd.0 [] 1194.042 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on back from osd.0 [] to osd.1 [] 1028.935 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_SLOW_PING_TIME_FRONT: Slow OSD heartbeats on front (longest 1565.748ms) 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.5 [] to osd.1 [] 1565.748 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.2 [] to osd.1 [] 1510.353 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.1 [] to osd.0 [] 1120.440 msec 2024-06-11T13:59:06.184 INFO:teuthology.orchestra.run.smithi069.stdout: Slow OSD heartbeats on front from osd.0 [] to osd.1 [] 1028.932 msec 2024-06-11T13:59:06.411 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:06 smithi069 ceph-mon[25054]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:06.411 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:06 smithi069 ceph-mon[25054]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:06.411 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:06 smithi069 ceph-mon[25054]: pgmap v9: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-11T13:59:06.411 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:06 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/3094241975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:06 smithi135 ceph-mon[27511]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:06 smithi135 ceph-mon[27511]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:06 smithi135 ceph-mon[27511]: pgmap v9: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-06-11T13:59:06.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:06 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/3094241975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:07.280 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:07 smithi069 ceph-mon[25054]: from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:07.280 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:07 smithi069 ceph-mon[25054]: from='client.? 172.21.15.69:0/81414658' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:59:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:07 smithi135 ceph-mon[27511]: from='client.14786 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:07 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/81414658' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:59:08.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:08 smithi069 ceph-mon[25054]: pgmap v10: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:08 smithi135 ceph-mon[27511]: pgmap v10: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:09.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:09 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:09.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:09 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:09.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:09 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:09.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:09 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:59:09.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:09 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:09.511 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:09 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:09.511 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:09 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:09.511 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:09 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:09.511 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:09 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:59:09.511 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:09 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:10.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:10 smithi069 ceph-mon[25054]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:59:10.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:10 smithi069 ceph-mon[25054]: pgmap v11: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:10 smithi135 ceph-mon[27511]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T13:59:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:10 smithi135 ceph-mon[27511]: pgmap v11: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:11 smithi069 ceph-mon[25054]: Upgrade: Updating mgr.smithi135.xcould 2024-06-11T13:59:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:11 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:59:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:11 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:59:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:11 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:11 smithi069 ceph-mon[25054]: Deploying daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:59:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:11 smithi135 ceph-mon[27511]: Upgrade: Updating mgr.smithi135.xcould 2024-06-11T13:59:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:11 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:59:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:11 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:59:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:11 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:11 smithi135 ceph-mon[27511]: Deploying daemon mgr.smithi135.xcould on smithi135 2024-06-11T13:59:12.443 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:12 smithi135 ceph-mon[27511]: pgmap v12: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:12.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:12 smithi069 ceph-mon[25054]: pgmap v12: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:13 smithi069 ceph-mon[25054]: Health check cleared: OSD_SLOW_PING_TIME_BACK (was: Slow OSD heartbeats on back (longest 1566.291ms)) 2024-06-11T13:59:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:13 smithi069 ceph-mon[25054]: Health check cleared: OSD_SLOW_PING_TIME_FRONT (was: Slow OSD heartbeats on front (longest 1565.748ms)) 2024-06-11T13:59:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:13 smithi069 ceph-mon[25054]: Cluster is now healthy 2024-06-11T13:59:13.521 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:13 smithi135 ceph-mon[27511]: Health check cleared: OSD_SLOW_PING_TIME_BACK (was: Slow OSD heartbeats on back (longest 1566.291ms)) 2024-06-11T13:59:13.521 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:13 smithi135 ceph-mon[27511]: Health check cleared: OSD_SLOW_PING_TIME_FRONT (was: Slow OSD heartbeats on front (longest 1565.748ms)) 2024-06-11T13:59:13.521 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:13 smithi135 ceph-mon[27511]: Cluster is now healthy 2024-06-11T13:59:14.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:14 smithi069 ceph-mon[25054]: pgmap v13: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:14.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:14 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:14.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:14 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:14.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:14 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:14.524 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:14 smithi135 ceph-mon[27511]: pgmap v13: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:14.525 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:14 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:14.525 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:14 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:14.525 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:14 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:16.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:16 smithi069 ceph-mon[25054]: pgmap v14: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:59:16.794 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:16 smithi135 ceph-mon[27511]: pgmap v14: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:59:18.310 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:18 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.310 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:18 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.310 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:18 smithi135 ceph-mon[27511]: pgmap v15: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:18.311 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:18 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.311 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:18 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:18 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.657 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:18 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:18 smithi069 ceph-mon[25054]: pgmap v15: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:18.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:18 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:18.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:18 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould started 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:19.653 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:19 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:19.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: Standby manager daemon smithi135.xcould started 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: from='mgr.? 172.21.15.135:0/2878794310' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:19.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:19 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:20.632 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:20 smithi135 ceph-mon[27511]: mgrmap e33: smithi069.iruzyw(active, since 26s), standbys: smithi135.xcould 2024-06-11T13:59:20.632 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:20 smithi135 ceph-mon[27511]: pgmap v16: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:20.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:20 smithi069 ceph-mon[25054]: mgrmap e33: smithi069.iruzyw(active, since 26s), standbys: smithi135.xcould 2024-06-11T13:59:20.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:20 smithi069 ceph-mon[25054]: pgmap v16: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: pgmap v17: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: Upgrade: Setting container_image for all mgr 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.iruzyw"}]: dispatch 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.iruzyw"}]': finished 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi135.xcould"}]: dispatch 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi135.xcould"}]': finished 2024-06-11T13:59:22.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:22 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: pgmap v17: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: Upgrade: Setting container_image for all mgr 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.iruzyw"}]: dispatch 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi069.iruzyw"}]': finished 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi135.xcould"}]: dispatch 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi135.xcould"}]': finished 2024-06-11T13:59:22.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:22 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: Upgrade: Updating mon.smithi069 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: Deploying daemon mon.smithi069 on smithi069 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: pgmap v18: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:23.987 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:23 smithi069 ceph-mon[25054]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: Upgrade: Updating mon.smithi069 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: Deploying daemon mon.smithi069 on smithi069 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: pgmap v18: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:23 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:24.542 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:24 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T13:59:24.828 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:24 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[25050]: 2024-06-11T13:59:24.542+0000 7f0aa926a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T13:59:24.828 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:24 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[25050]: 2024-06-11T13:59:24.542+0000 7f0aa926a700 -1 mon.smithi069@0(leader) e2 *** Got Signal Terminated *** 2024-06-11T13:59:24.828 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:24 smithi069 podman[66719]: 2024-06-11 13:59:24.613997721 +0000 UTC m=+0.191390916 container died 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, vcs-type=git, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, distribution-scope=public, name=centos-stream, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.openshift.expose-services=, release=754, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc.) 2024-06-11T13:59:25.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:24 smithi069 podman[66719]: 2024-06-11 13:59:24.975916789 +0000 UTC m=+0.553309981 container cleanup 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, RELEASE=HEAD, name=centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.k8s.display-name=CentOS Stream 8) 2024-06-11T13:59:25.217 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:24 smithi069 bash[66719]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069 2024-06-11T13:59:25.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 podman[66730]: 2024-06-11 13:59:25.218890515 +0000 UTC m=+0.606726130 container remove 0c4fab281c29bca9fb993fa98fa1ac893cfed044478fd3f07c96de08e7cab4f1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, ceph=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container) 2024-06-11T13:59:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service: Deactivated successfully. 2024-06-11T13:59:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 systemd[1]: Stopped Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:59:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service: Consumed 22.560s CPU time. 2024-06-11T13:59:25.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 systemd[1]: Starting Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T13:59:25.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 podman[66873]: 2024-06-11 13:59:25.786593573 +0000 UTC m=+0.112428742 container create f050c71c12424936a450017e1d40523059426b857852a6916b764e6702cfe54f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3) 2024-06-11T13:59:25.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:25 smithi069 podman[66873]: 2024-06-11 13:59:25.691493662 +0000 UTC m=+0.017328836 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 podman[66873]: 2024-06-11 13:59:26.030990258 +0000 UTC m=+0.356825433 container init f050c71c12424936a450017e1d40523059426b857852a6916b764e6702cfe54f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, ceph=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0) 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 podman[66873]: 2024-06-11 13:59:26.034132679 +0000 UTC m=+0.359967853 container start f050c71c12424936a450017e1d40523059426b857852a6916b764e6702cfe54f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: set uid:gid to 167:167 (ceph:ceph) 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc), process ceph-mon, pid 2 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: pidfile_write: ignore empty --pid-file 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: load: jerasure load: lrc 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: RocksDB version: 7.9.2 2024-06-11T13:59:26.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Git sha 0 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Compile date 2024-06-10 15:42:49 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: DB SUMMARY 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: DB Session ID: 38J7JX00S9FOJRDI3W34 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: CURRENT file: CURRENT 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: IDENTITY file: IDENTITY 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1905 Bytes 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi069/store.db dir, Total Num: 1, files: 000045.sst 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi069/store.db: 000043.log size: 590635 ; 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.error_if_exists: 0 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.create_if_missing: 0 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.paranoid_checks: 1 2024-06-11T13:59:26.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-11T13:59:26.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-11T13:59:26.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-11T13:59:26.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.env: 0x55cf8c079740 2024-06-11T13:59:26.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.fs: PosixFileSystem 2024-06-11T13:59:26.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.info_log: 0x55cf8ca83240 2024-06-11T13:59:26.250 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_file_opening_threads: 16 2024-06-11T13:59:26.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.statistics: (nil) 2024-06-11T13:59:26.251 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.use_fsync: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_log_file_size: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.keep_log_file_num: 1000 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.recycle_log_file_num: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.allow_fallocate: 1 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.allow_mmap_reads: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.allow_mmap_writes: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.use_direct_reads: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.create_missing_column_families: 0 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.db_log_dir: 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.wal_dir: 2024-06-11T13:59:26.252 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.advise_random_on_open: 1 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.db_write_buffer_size: 0 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.write_buffer_manager: 0x55cf8ca921e0 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.rate_limiter: (nil) 2024-06-11T13:59:26.253 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.wal_recovery_mode: 2 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.enable_thread_tracking: 0 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.enable_pipelined_write: 0 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.unordered_write: 0 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.row_cache: None 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.wal_filter: None 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.allow_ingest_behind: 0 2024-06-11T13:59:26.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.two_write_queues: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.manual_wal_flush: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.wal_compression: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.atomic_flush: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.log_readahead_size: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.best_efforts_recovery: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.allow_data_in_errors: 0 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.db_host_id: __hostname__ 2024-06-11T13:59:26.255 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_background_jobs: 2 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_background_compactions: -1 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_subcompactions: 1 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_total_wal_size: 0 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_open_files: -1 2024-06-11T13:59:26.256 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bytes_per_sync: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_readahead_size: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_background_flushes: -1 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Compression algorithms supported: 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kZSTD supported: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kXpressCompression supported: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kBZip2Compression supported: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kLZ4Compression supported: 1 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kZlibCompression supported: 1 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kLZ4HCCompression supported: 1 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: kSnappyCompression supported: 1 2024-06-11T13:59:26.257 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000015 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.merge_operator: 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_filter: None 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_filter_factory: None 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.sst_partitioner_factory: None 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55cf8ca82ec0) 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks: 1 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-11T13:59:26.258 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_top_level_index_and_filter: 1 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_type: 0 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_index_type: 0 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_shortening: 1 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: checksum: 4 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: no_block_cache: 0 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache: 0x55cf8ca79090 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_name: BinnedLRUCache 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_options: 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: capacity : 536870912 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_shard_bits : 4 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: strict_capacity_limit : 0 2024-06-11T13:59:26.259 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: high_pri_pool_ratio: 0.000 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_compressed: (nil) 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: persistent_cache: (nil) 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size: 4096 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size_deviation: 10 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_restart_interval: 16 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_block_restart_interval: 1 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_block_size: 4096 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: partition_filters: 0 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: use_delta_encoding: 1 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: filter_policy: bloomfilter 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: whole_key_filtering: 1 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: verify_compression: 0 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: read_amp_bytes_per_bit: 0 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: format_version: 5 2024-06-11T13:59:26.260 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_index_compression: 1 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_align: 0 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_auto_readahead_size: 262144 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: prepopulate_block_cache: 0 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: initial_auto_readahead_size: 8192 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_file_reads_for_auto_readahead: 2 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.write_buffer_size: 33554432 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_write_buffer_number: 2 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression: NoCompression 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression: Disabled 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.prefix_extractor: nullptr 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.num_levels: 7 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-11T13:59:26.261 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.level: 32767 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.strategy: 0 2024-06-11T13:59:26.262 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.enabled: false 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.target_file_size_base: 67108864 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-11T13:59:26.263 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-11T13:59:26.264 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.arena_block_size: 1048576 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.disable_auto_compactions: 0 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-11T13:59:26.265 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.table_properties_collectors: 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.inplace_update_support: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.bloom_locality: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.max_successive_merges: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.paranoid_file_checks: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.force_consistency_checks: 1 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.report_bg_io_stats: 0 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.ttl: 2592000 2024-06-11T13:59:26.266 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.enable_blob_files: false 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.min_blob_size: 0 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.blob_file_size: 268435456 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.blob_file_starting_level: 0 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 45.sst 2024-06-11T13:59:26.267 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 47, last_sequence is 21058, log_number is 43,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 43 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5f0d5c3a-0c28-4198-beae-7d8097c8189b 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718114366061497, "job": 1, "event": "recovery_started", "wal_files": [43]} 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #43 mode 2 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718114366064970, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 48, "file_size": 550086, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 21059, "largest_seqno": 21355, "table_properties": {"data_size": 547740, "index_size": 948, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3957, "raw_average_key_size": 26, "raw_value_size": 544194, "raw_average_value_size": 3603, "num_data_blocks": 42, "num_entries": 151, "num_filter_entries": 151, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718114366, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5f0d5c3a-0c28-4198-beae-7d8097c8189b", "db_session_id": "38J7JX00S9FOJRDI3W34", "orig_file_number": 48, "seqno_to_time_mapping": "N/A"}} 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718114366065098, "job": 1, "event": "recovery_finished"} 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/version_set.cc:5047] Creating manifest 50 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi069/store.db/000043.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55cf8caa6e00 2024-06-11T13:59:26.268 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: DB pointer 0x55cf8cb9c000 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** DB Stats ** 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-11T13:59:26.269 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 1/0 537.19 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 240.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L6 1/0 9.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 9.57 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 240.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 240.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 240.1 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:59:26.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 62.68 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 62.68 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Block cache BinnedLRUCache@0x55cf8ca79090#2 capacity: 512.00 MB usage: 1.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: starting mon.smithi069 rank 0 at public addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] at bind addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi069 fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:26.271 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???) e2 preinit fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).mds e6 new map 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).mds e6 print_map 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: e6 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: legacy client fscid: 1 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Filesystem 'foofs' (1) 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: fs_name foofs 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: epoch 5 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: created 2024-06-11T13:42:08.920168+0000 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: modified 2024-06-11T13:42:13.942143+0000 2024-06-11T13:59:26.272 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: tableserver 0 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: root 0 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: session_timeout 60 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: session_autoclose 300 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_file_size 1099511627776 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_xattr_size 65536 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: required_client_features {} 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: last_failure 0 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: last_failure_osd_epoch 0 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: max_mds 1 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: in 0 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: up {0=14498} 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: failed 2024-06-11T13:59:26.273 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: damaged 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: stopped 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_pools [3] 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_pool 2 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: inline_data disabled 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: balancer 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: bal_rank_mask -1 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: standby_count_wanted 1 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: qdb_cluster leader: 0 members: 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: [mds.foofs.smithi069.nsgdqs{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.69:6834/2689424606,v1:172.21.15.69:6835/2689424606] compat {c=[1],r=[1],i=[7ff]}] 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Standby daemons: 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-06-11T13:59:26.274 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: [mds.foofs.smithi135.wtrawd{-1:24295} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] compat {c=[1],r=[1],i=[7ff]}] 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 ceph-mon[66891]: mon.smithi069@-1(???).paxosservice(auth 1..33) refresh upgraded, format 0 -> 3 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 bash[66873]: f050c71c12424936a450017e1d40523059426b857852a6916b764e6702cfe54f 2024-06-11T13:59:26.275 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:26 smithi069 systemd[1]: Started Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: pgmap v19: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: pgmap v20: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: mon.smithi069 calling monitor election 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: mon.smithi069 is new leader, mons smithi069,smithi135 in quorum (ranks 0,1) 2024-06-11T13:59:29.407 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: monmap epoch 2 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: last_changed 2024-06-11T13:37:11.616682+0000 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: created 2024-06-11T13:34:53.486112+0000 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: min_mon_release 17 (quincy) 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: election_strategy: 1 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: 1: [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon.smithi135 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: osdmap e83: 8 total, 8 up, 8 in 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: mgrmap e33: smithi069.iruzyw(active, since 36s), standbys: smithi135.xcould 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: overall HEALTH_OK 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: from='mgr.14740 ' entity='' 2024-06-11T13:59:29.408 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:29 smithi135 ceph-mon[27511]: mgrmap e34: smithi069.iruzyw(active, since 36s), standbys: smithi135.xcould 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: from='mgr.14740 172.21.15.69:0/1295974018' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: pgmap v19: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: pgmap v20: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: mon.smithi069 calling monitor election 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: mon.smithi069 is new leader, mons smithi069,smithi135 in quorum (ranks 0,1) 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: monmap epoch 2 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: last_changed 2024-06-11T13:37:11.616682+0000 2024-06-11T13:59:29.446 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: created 2024-06-11T13:34:53.486112+0000 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: min_mon_release 17 (quincy) 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: election_strategy: 1 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: 1: [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon.smithi135 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: osdmap e83: 8 total, 8 up, 8 in 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: mgrmap e33: smithi069.iruzyw(active, since 36s), standbys: smithi135.xcould 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: overall HEALTH_OK 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: from='mgr.14740 ' entity='' 2024-06-11T13:59:29.447 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:29 smithi069 ceph-mon[66891]: mgrmap e34: smithi069.iruzyw(active, since 36s), standbys: smithi135.xcould 2024-06-11T13:59:31.410 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:31 smithi135 ceph-mon[27511]: mgrmap e35: smithi069.iruzyw(active, since 37s), standbys: smithi135.xcould 2024-06-11T13:59:31.439 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:31 smithi069 ceph-mon[66891]: mgrmap e35: smithi069.iruzyw(active, since 37s), standbys: smithi135.xcould 2024-06-11T13:59:34.131 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:59:34.131 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: Standby manager daemon smithi135.xcould started 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: Active manager daemon smithi069.iruzyw restarted 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: osdmap e84: 8 total, 8 up, 8 in 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: mgrmap e36: smithi069.iruzyw(active, starting, since 0.00490012s), standbys: smithi135.xcould 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:59:34.132 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:59:34.133 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:59:34.134 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:59:34.134 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:59:34.134 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:59:34.134 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:59:34.134 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:59:34.134 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:59:34.135 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:34 smithi135 ceph-mon[27511]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:59:34.173 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: Standby manager daemon smithi135.xcould restarted 2024-06-11T13:59:34.173 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: Standby manager daemon smithi135.xcould started 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/crt"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi135.xcould/key"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.? 172.21.15.135:0/1692292417' entity='mgr.smithi135.xcould' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: Active manager daemon smithi069.iruzyw restarted 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: Activating manager daemon smithi069.iruzyw 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: osdmap e84: 8 total, 8 up, 8 in 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: mgrmap e36: smithi069.iruzyw(active, starting, since 0.00490012s), standbys: smithi135.xcould 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T13:59:34.174 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi069.iruzyw", "id": "smithi069.iruzyw"}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr metadata", "who": "smithi135.xcould", "id": "smithi135.xcould"}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-06-11T13:59:34.175 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:34 smithi069 ceph-mon[66891]: Manager daemon smithi069.iruzyw is now available 2024-06-11T13:59:35.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:35.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:35.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:59:35.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:59:35.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:35 smithi069 ceph-mon[66891]: mgrmap e37: smithi069.iruzyw(active, since 1.00965s), standbys: smithi135.xcould 2024-06-11T13:59:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:35 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:35 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:35 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/mirror_snapshot_schedule"}]: dispatch 2024-06-11T13:59:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:35 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.iruzyw/trash_purge_schedule"}]: dispatch 2024-06-11T13:59:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:35 smithi135 ceph-mon[27511]: mgrmap e37: smithi069.iruzyw(active, since 1.00965s), standbys: smithi135.xcould 2024-06-11T13:59:36.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:36 smithi069 ceph-mon[66891]: pgmap v3: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:36.486 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:36 smithi135 ceph-mon[27511]: pgmap v3: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:36.542 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (71s) 17s ago 23m 16.5M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (23m) 17s ago 23m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (22m) 19s ago 22m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (67s) 17s ago 22m 43.1M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (17m) 17s ago 17m 39.5M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (17m) 19s ago 17m 18.4M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (54s) 17s ago 24m 491M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (23s) 19s ago 22m 52.9M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (24m) 17s ago 24m 75.7M 2048M 17.2.0 e1d6a67b021e 0c4fab281c29 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (22m) 19s ago 22m 56.9M 2048M 17.2.0 e1d6a67b021e 92b798825496 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (17m) 17s ago 17m 278M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (16m) 19s ago 16m 87.1M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T13:59:37.213 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (65s) 17s ago 22m 8418k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (59s) 19s ago 22m 14.6M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (21m) 17s ago 21m 499M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (21m) 17s ago 21m 411M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (20m) 17s ago 20m 528M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (20m) 17s ago 20m 414M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (20m) 19s ago 20m 458M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (19m) 19s ago 19m 533M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (19m) 19s ago 19m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (18m) 19s ago 18m 688M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T13:59:37.214 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (28s) 17s ago 22m 51.1M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: [11/Jun/2024:13:59:35] ENGINE Bus STARTING 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: [11/Jun/2024:13:59:35] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: [11/Jun/2024:13:59:35] ENGINE Error in HTTPServer.serve 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Traceback (most recent call last): 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._connections.run(self.expiration_interval) 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._run(expiration_interval) 2024-06-11T13:59:37.484 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: s = self.context.wrap_socket( 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: return self.sslsocket_class._create( 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self.do_handshake() 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: self._sslobj.do_handshake() 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: [11/Jun/2024:13:59:35] ENGINE Serving on http://172.21.15.69:8765 2024-06-11T13:59:37.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: [11/Jun/2024:13:59:35] ENGINE Bus STARTED 2024-06-11T13:59:37.486 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: pgmap v4: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:37.486 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:37 smithi069 ceph-mon[66891]: mgrmap e38: smithi069.iruzyw(active, since 2s), standbys: smithi135.xcould 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: [11/Jun/2024:13:59:35] ENGINE Bus STARTING 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: [11/Jun/2024:13:59:35] ENGINE Serving on https://172.21.15.69:7150 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: [11/Jun/2024:13:59:35] ENGINE Error in HTTPServer.serve 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Traceback (most recent call last): 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._connections.run(self.expiration_interval) 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._run(expiration_interval) 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-06-11T13:59:37.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: s = self.context.wrap_socket( 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: return self.sslsocket_class._create( 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self.do_handshake() 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: self._sslobj.do_handshake() 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1129) 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: [11/Jun/2024:13:59:35] ENGINE Serving on http://172.21.15.69:8765 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: [11/Jun/2024:13:59:35] ENGINE Bus STARTED 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: pgmap v4: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:37.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:37 smithi135 ceph-mon[27511]: mgrmap e38: smithi069.iruzyw(active, since 2s), standbys: smithi135.xcould 2024-06-11T13:59:37.615 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:59:37.615 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T13:59:37.615 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-11T13:59:37.615 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 1 2024-06-11T13:59:37.615 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 3 2024-06-11T13:59:37.616 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T13:59:37.617 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:59:37.968 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T13:59:37.968 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "2/23 daemons upgraded", 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T13:59:37.969 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T13:59:38.235 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:38 smithi069 ceph-mon[66891]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:38.236 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:38 smithi069 ceph-mon[66891]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:38.236 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:38 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/442707101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:38.236 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:38.236 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:38.398 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T13:59:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:38 smithi135 ceph-mon[27511]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:38 smithi135 ceph-mon[27511]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:38 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/442707101' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:38 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:38 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:39.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:39.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:39.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: pgmap v5: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:39.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2574719103' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:59:39.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:39.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:39.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:39 smithi069 ceph-mon[66891]: mgrmap e39: smithi069.iruzyw(active, since 4s), standbys: smithi135.xcould 2024-06-11T13:59:39.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T13:59:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: pgmap v5: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: from='client.? 172.21.15.69:0/2574719103' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T13:59:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:39.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:39 smithi135 ceph-mon[27511]: mgrmap e39: smithi069.iruzyw(active, since 4s), standbys: smithi135.xcould 2024-06-11T13:59:41.424 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:41 smithi135 ceph-mon[27511]: pgmap v6: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:41.424 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:41 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.424 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:41 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.424 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:41 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.424 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:41 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:41 smithi069 ceph-mon[66891]: pgmap v6: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-06-11T13:59:41.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: pgmap v7: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:43.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:42 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.275 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.275 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.conf 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.conf 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: pgmap v7: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.conf 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:43.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:42 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:44.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:44.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:44.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:44.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:44.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:44.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:44.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: Updating smithi135:/etc/ceph/ceph.client.admin.keyring 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: Updating smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: Updating smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/config/ceph.client.admin.keyring 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T13:59:44.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:43 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-06-11T13:59:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:44 smithi069 ceph-mon[66891]: pgmap v8: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-11T13:59:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:59:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:59:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:45.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:44 smithi135 ceph-mon[27511]: pgmap v8: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-06-11T13:59:45.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:44 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:45.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:44 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:59:45.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:44 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:59:45.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:44 smithi135 ceph-mon[27511]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:46.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:45 smithi135 ceph-mon[27511]: Upgrade: Updating mon.smithi135 2024-06-11T13:59:46.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:45 smithi135 ceph-mon[27511]: Deploying daemon mon.smithi135 on smithi135 2024-06-11T13:59:46.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:45 smithi135 systemd[1]: Stopping Ceph mon.smithi135 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T13:59:46.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:45 smithi069 ceph-mon[66891]: Upgrade: Updating mon.smithi135 2024-06-11T13:59:46.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:45 smithi069 ceph-mon[66891]: Deploying daemon mon.smithi135 on smithi135 2024-06-11T13:59:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:46 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135[27492]: 2024-06-11T13:59:46.157+0000 7f0b73f58700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi135 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T13:59:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:46 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135[27492]: 2024-06-11T13:59:46.157+0000 7f0b73f58700 -1 mon.smithi135@1(peon) e2 *** Got Signal Terminated *** 2024-06-11T13:59:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:46 smithi135 podman[53625]: 2024-06-11 13:59:46.229632831 +0000 UTC m=+0.186262532 container died 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, GIT_CLEAN=True, ceph=True, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc.) 2024-06-11T13:59:47.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:46 smithi135 podman[53625]: 2024-06-11 13:59:46.749165942 +0000 UTC m=+0.705795687 container cleanup 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, version=8, vcs-type=git, RELEASE=HEAD, GIT_CLEAN=True, io.openshift.expose-services=, architecture=x86_64, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream) 2024-06-11T13:59:47.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:46 smithi135 bash[53625]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135 2024-06-11T13:59:47.395 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 podman[53640]: 2024-06-11 13:59:47.092228489 +0000 UTC m=+0.854959087 container remove 92b7988254967de92252e48f70a081d73af9d5165ad469d28967614d29e0fbdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, distribution-scope=public, name=centos-stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.openshift.expose-services=, ceph=True, com.redhat.component=centos-stream-container, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-11T13:59:47.395 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi135.service: Deactivated successfully. 2024-06-11T13:59:47.395 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 systemd[1]: Stopped Ceph mon.smithi135 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:59:47.395 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi135.service: Consumed 10.115s CPU time. 2024-06-11T13:59:47.740 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 systemd[1]: Starting Ceph mon.smithi135 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T13:59:47.740 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 podman[53764]: 2024-06-11 13:59:47.515769451 +0000 UTC m=+0.017089116 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T13:59:47.740 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 podman[53764]: 2024-06-11 13:59:47.634922062 +0000 UTC m=+0.136241712 container create a50d298981f4958faefeb464b74ca7a52ae2dbfb65e967c5c2594cb8603704c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, io.buildah.version=1.35.3) 2024-06-11T13:59:48.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 podman[53764]: 2024-06-11 13:59:47.870990611 +0000 UTC m=+0.372310259 container init a50d298981f4958faefeb464b74ca7a52ae2dbfb65e967c5c2594cb8603704c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T13:59:48.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 podman[53764]: 2024-06-11 13:59:47.87419979 +0000 UTC m=+0.375519448 container start a50d298981f4958faefeb464b74ca7a52ae2dbfb65e967c5c2594cb8603704c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T13:59:48.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: set uid:gid to 167:167 (ceph:ceph) 2024-06-11T13:59:48.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc), process ceph-mon, pid 2 2024-06-11T13:59:48.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: pidfile_write: ignore empty --pid-file 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: load: jerasure load: lrc 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: RocksDB version: 7.9.2 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Git sha 0 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Compile date 2024-06-10 15:42:49 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: DB SUMMARY 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: DB Session ID: QGDV1QV3X3MLMDC0N3I5 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: CURRENT file: CURRENT 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: IDENTITY file: IDENTITY 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: MANIFEST file: MANIFEST-000009 size: 2221 Bytes 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi135/store.db dir, Total Num: 1, files: 000045.sst 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi135/store.db: 000043.log size: 341510 ; 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.error_if_exists: 0 2024-06-11T13:59:48.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.create_if_missing: 0 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.paranoid_checks: 1 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.flush_verify_memtable_count: 1 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.env: 0x55fd69327740 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.fs: PosixFileSystem 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.info_log: 0x55fd6a02d240 2024-06-11T13:59:48.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_file_opening_threads: 16 2024-06-11T13:59:48.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.statistics: (nil) 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.use_fsync: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_log_file_size: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.log_file_time_to_roll: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.keep_log_file_num: 1000 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.recycle_log_file_num: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.allow_fallocate: 1 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.allow_mmap_reads: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.allow_mmap_writes: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.use_direct_reads: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.create_missing_column_families: 0 2024-06-11T13:59:48.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.db_log_dir: 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.wal_dir: 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.table_cache_numshardbits: 6 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.WAL_ttl_seconds: 0 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.WAL_size_limit_MB: 0 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.is_fd_close_on_exec: 1 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.advise_random_on_open: 1 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.db_write_buffer_size: 0 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.write_buffer_manager: 0x55fd6a03c1e0 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.use_adaptive_mutex: 0 2024-06-11T13:59:48.052 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.rate_limiter: (nil) 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.wal_recovery_mode: 2 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.enable_thread_tracking: 0 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.enable_pipelined_write: 0 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.unordered_write: 0 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.row_cache: None 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.wal_filter: None 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.allow_ingest_behind: 0 2024-06-11T13:59:48.053 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.two_write_queues: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.manual_wal_flush: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.wal_compression: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.atomic_flush: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.persist_stats_to_disk: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.write_dbid_to_manifest: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.log_readahead_size: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.best_efforts_recovery: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.allow_data_in_errors: 0 2024-06-11T13:59:48.054 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.db_host_id: __hostname__ 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.enforce_single_del_contracts: true 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_background_jobs: 2 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_background_compactions: -1 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_subcompactions: 1 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.delayed_write_rate : 16777216 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_total_wal_size: 0 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.stats_dump_period_sec: 600 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.stats_persist_period_sec: 600 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_open_files: -1 2024-06-11T13:59:48.055 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bytes_per_sync: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.wal_bytes_per_sync: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.strict_bytes_per_sync: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_readahead_size: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_background_flushes: -1 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Compression algorithms supported: 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kZSTD supported: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kXpressCompression supported: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kBZip2Compression supported: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kLZ4Compression supported: 1 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kZlibCompression supported: 1 2024-06-11T13:59:48.056 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kLZ4HCCompression supported: 1 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: kSnappyCompression supported: 1 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Fast CRC32 supported: Supported on x86 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: DMutex implementation: pthread_mutex_t 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi135/store.db/MANIFEST-000009 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.merge_operator: 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_filter: None 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_filter_factory: None 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.sst_partitioner_factory: None 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.memtable_factory: SkipListFactory 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.table_factory: BlockBasedTable 2024-06-11T13:59:48.057 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fd6a02cec0) 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: cache_index_and_filter_blocks: 1 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: pin_top_level_index_and_filter: 1 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: index_type: 0 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: data_block_index_type: 0 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: index_shortening: 1 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: data_block_hash_table_util_ratio: 0.750000 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: checksum: 4 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: no_block_cache: 0 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache: 0x55fd6a023090 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache_name: BinnedLRUCache 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache_options: 2024-06-11T13:59:48.058 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: capacity : 536870912 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: num_shard_bits : 4 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: strict_capacity_limit : 0 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: high_pri_pool_ratio: 0.000 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_cache_compressed: (nil) 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: persistent_cache: (nil) 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_size: 4096 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_size_deviation: 10 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_restart_interval: 16 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: index_block_restart_interval: 1 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: metadata_block_size: 4096 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: partition_filters: 0 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: use_delta_encoding: 1 2024-06-11T13:59:48.059 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: filter_policy: bloomfilter 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: whole_key_filtering: 1 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: verify_compression: 0 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: read_amp_bytes_per_bit: 0 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: format_version: 5 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: enable_index_compression: 1 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: block_align: 0 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: max_auto_readahead_size: 262144 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: prepopulate_block_cache: 0 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: initial_auto_readahead_size: 8192 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: num_file_reads_for_auto_readahead: 2 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.write_buffer_size: 33554432 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_write_buffer_number: 2 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression: NoCompression 2024-06-11T13:59:48.060 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression: Disabled 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.prefix_extractor: nullptr 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.num_levels: 7 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-06-11T13:59:48.061 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.window_bits: -14 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.level: 32767 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.strategy: 0 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.enabled: false 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-06-11T13:59:48.062 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.target_file_size_base: 67108864 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.target_file_size_multiplier: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-06-11T13:59:48.063 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.arena_block_size: 1048576 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.disable_auto_compactions: 0 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-06-11T13:59:48.064 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.table_properties_collectors: 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.inplace_update_support: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.inplace_update_num_locks: 10000 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.memtable_huge_page_size: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.bloom_locality: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.max_successive_merges: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.optimize_filters_for_hits: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.paranoid_file_checks: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.force_consistency_checks: 1 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.report_bg_io_stats: 0 2024-06-11T13:59:48.065 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.ttl: 2592000 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.periodic_compaction_seconds: 0 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.enable_blob_files: false 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.min_blob_size: 0 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.blob_file_size: 268435456 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.blob_compression_type: NoCompression 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.enable_blob_garbage_collection: false 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.blob_file_starting_level: 0 2024-06-11T13:59:48.066 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 45.sst 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi135/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 47, last_sequence is 23640, log_number is 43,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 43 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 457848ae-49a0-4194-a73c-ca7939b539dd 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718114387901243, "job": 1, "event": "recovery_started", "wal_files": [43]} 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #43 mode 2 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718114387903138, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 48, "file_size": 214435, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 23645, "largest_seqno": 23800, "table_properties": {"data_size": 212628, "index_size": 539, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2169, "raw_average_key_size": 26, "raw_value_size": 210683, "raw_average_value_size": 2538, "num_data_blocks": 22, "num_entries": 83, "num_filter_entries": 83, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1718114387, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "457848ae-49a0-4194-a73c-ca7939b539dd", "db_session_id": "QGDV1QV3X3MLMDC0N3I5", "orig_file_number": 48, "seqno_to_time_mapping": "N/A"}} 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: EVENT_LOG_v1 {"time_micros": 1718114387903262, "job": 1, "event": "recovery_finished"} 2024-06-11T13:59:48.067 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/version_set.cc:5047] Creating manifest 50 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi135/store.db/000043.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fd6a050e00 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: DB pointer 0x55fd6a146000 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** DB Stats ** 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-06-11T13:59:48.068 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** Compaction Stats [default] ** 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: L0 1/0 209.41 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 181.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: L6 1/0 10.10 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Sum 2/0 10.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 181.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 181.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** Compaction Stats [default] ** 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-06-11T13:59:48.069 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 181.3 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: AddFile(Keys): cumulative 0, interval 0 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Cumulative compaction: 0.00 GB write, 43.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Interval compaction: 0.00 GB write, 43.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-06-11T13:59:48.070 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Block cache BinnedLRUCache@0x55fd6a023090#2 capacity: 512.00 MB usage: 0.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,0.64 KB,0.00012219%) Misc(1,0.00 KB,0%) 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: ** File Read Latency Histogram By Level [default] ** 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: starting mon.smithi135 rank 1 at public addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] at bind addrs [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi135 fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???) e2 preinit fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).mds e6 new map 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).mds e6 print_map 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: e6 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: legacy client fscid: 1 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Filesystem 'foofs' (1) 2024-06-11T13:59:48.071 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: fs_name foofs 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: epoch 5 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: created 2024-06-11T13:42:08.920168+0000 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: modified 2024-06-11T13:42:13.942143+0000 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: tableserver 0 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: root 0 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: session_timeout 60 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: session_autoclose 300 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: max_file_size 1099511627776 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: max_xattr_size 65536 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: required_client_features {} 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: last_failure 0 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: last_failure_osd_epoch 0 2024-06-11T13:59:48.072 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: max_mds 1 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: in 0 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: up {0=14498} 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: failed 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: damaged 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: stopped 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: data_pools [3] 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: metadata_pool 2 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: inline_data disabled 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: balancer 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: bal_rank_mask -1 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: standby_count_wanted 1 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: qdb_cluster leader: 0 members: 2024-06-11T13:59:48.073 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: [mds.foofs.smithi069.nsgdqs{0:14498} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.69:6834/2689424606,v1:172.21.15.69:6835/2689424606] compat {c=[1],r=[1],i=[7ff]}] 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: Standby daemons: 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout: [mds.foofs.smithi135.wtrawd{-1:24295} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] compat {c=[1],r=[1],i=[7ff]}] 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 ceph-mon[53782]: mon.smithi135@-1(???).paxosservice(auth 1..34) refresh upgraded, format 0 -> 3 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 bash[53764]: a50d298981f4958faefeb464b74ca7a52ae2dbfb65e967c5c2594cb8603704c8 2024-06-11T13:59:48.074 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:47 smithi135 systemd[1]: Started Ceph mon.smithi135 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T13:59:49.678 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:59:49.678 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: mon.smithi069 calling monitor election 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: mon.smithi135 calling monitor election 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: mon.smithi069 is new leader, mons smithi069,smithi135 in quorum (ranks 0,1) 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: monmap epoch 3 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: last_changed 2024-06-11T13:59:48.310789+0000 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: created 2024-06-11T13:34:53.486112+0000 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: min_mon_release 19 (squid) 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: election_strategy: 1 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: 1: [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon.smithi135 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T13:59:49.679 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: osdmap e84: 8 total, 8 up, 8 in 2024-06-11T13:59:49.680 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: mgrmap e39: smithi069.iruzyw(active, since 14s), standbys: smithi135.xcould 2024-06-11T13:59:49.680 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: overall HEALTH_OK 2024-06-11T13:59:49.680 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:49.680 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:49.680 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:49.680 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: mon.smithi069 calling monitor election 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: mon.smithi135 calling monitor election 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mon metadata", "id": "smithi135"}]: dispatch 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: mon.smithi069 is new leader, mons smithi069,smithi135 in quorum (ranks 0,1) 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: monmap epoch 3 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: last_changed 2024-06-11T13:59:48.310789+0000 2024-06-11T13:59:49.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: created 2024-06-11T13:34:53.486112+0000 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: min_mon_release 19 (squid) 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: election_strategy: 1 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: 0: [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon.smithi069 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: 1: [v2:172.21.15.135:3300/0,v1:172.21.15.135:6789/0] mon.smithi135 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: osdmap e84: 8 total, 8 up, 8 in 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: mgrmap e39: smithi069.iruzyw(active, since 14s), standbys: smithi135.xcould 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: overall HEALTH_OK 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T13:59:49.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T13:59:50.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:50 smithi135 ceph-mon[53782]: pgmap v11: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:50.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:50 smithi069 ceph-mon[66891]: pgmap v11: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:52 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:52 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:52 smithi135 ceph-mon[53782]: pgmap v12: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:52 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:52 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:52 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:52 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:52 smithi069 ceph-mon[66891]: pgmap v12: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-06-11T13:59:52.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:52 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:52.915 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:52 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:55.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:55 smithi135 ceph-mon[53782]: pgmap v13: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:55.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:55 smithi069 ceph-mon[66891]: pgmap v13: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: Reconfiguring mon.smithi069 (monmap changed)... 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: Reconfiguring daemon mon.smithi069 on smithi069 2024-06-11T13:59:56.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:56 smithi069 ceph-mon[66891]: pgmap v14: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: Reconfiguring mon.smithi069 (monmap changed)... 2024-06-11T13:59:56.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T13:59:56.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T13:59:56.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:56.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: Reconfiguring daemon mon.smithi069 on smithi069 2024-06-11T13:59:56.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:56 smithi135 ceph-mon[53782]: pgmap v14: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:57.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: Reconfiguring mgr.smithi069.iruzyw (monmap changed)... 2024-06-11T13:59:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:59:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:59:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:57.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:57 smithi069 ceph-mon[66891]: Reconfiguring daemon mgr.smithi069.iruzyw on smithi069 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: Reconfiguring mgr.smithi069.iruzyw (monmap changed)... 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi069.iruzyw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:57 smithi135 ceph-mon[53782]: Reconfiguring daemon mgr.smithi069.iruzyw on smithi069 2024-06-11T13:59:58.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:58 smithi069 ceph-mon[66891]: pgmap v15: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:58.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:58.958 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:58.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:59:58.959 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:58 smithi135 ceph-mon[53782]: pgmap v15: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T13:59:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T13:59:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T13:59:59.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:59 smithi069 ceph-mon[66891]: Reconfiguring crash.smithi069 (monmap changed)... 2024-06-11T13:59:59.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:59 smithi069 ceph-mon[66891]: Reconfiguring daemon crash.smithi069 on smithi069 2024-06-11T13:59:59.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:59 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:59.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:59 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T13:59:59.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:59 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-11T13:59:59.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 13:59:59 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:59 smithi135 ceph-mon[53782]: Reconfiguring crash.smithi069 (monmap changed)... 2024-06-11T14:00:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:59 smithi135 ceph-mon[53782]: Reconfiguring daemon crash.smithi069 on smithi069 2024-06-11T14:00:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:59 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:59 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:59 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-11T14:00:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 13:59:59 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:01.086 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:00 smithi069 ceph-mon[66891]: Reconfiguring osd.0 (monmap changed)... 2024-06-11T14:00:01.086 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:00 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.0 on smithi069 2024-06-11T14:00:01.086 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:00 smithi069 ceph-mon[66891]: overall HEALTH_OK 2024-06-11T14:00:01.087 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:00 smithi069 ceph-mon[66891]: pgmap v16: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:01.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:00 smithi135 ceph-mon[53782]: Reconfiguring osd.0 (monmap changed)... 2024-06-11T14:00:01.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:00 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.0 on smithi069 2024-06-11T14:00:01.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:00 smithi135 ceph-mon[53782]: overall HEALTH_OK 2024-06-11T14:00:01.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:00 smithi135 ceph-mon[53782]: pgmap v16: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:02.411 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:02 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:02.412 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:02 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:02.412 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:02 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-11T14:00:02.412 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:02 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:02 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:02 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:02 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-11T14:00:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:02 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:03.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: Reconfiguring osd.1 (monmap changed)... 2024-06-11T14:00:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.1 on smithi069 2024-06-11T14:00:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: pgmap v17: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-11T14:00:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: Reconfiguring osd.1 (monmap changed)... 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.1 on smithi069 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: pgmap v17: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-11T14:00:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:04.458 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: Reconfiguring osd.2 (monmap changed)... 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.2 on smithi069 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: Reconfiguring osd.3 (monmap changed)... 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.3 on smithi069 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: pgmap v18: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:04.459 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: Reconfiguring osd.2 (monmap changed)... 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.2 on smithi069 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: Reconfiguring osd.3 (monmap changed)... 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.3 on smithi069 2024-06-11T14:00:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: pgmap v18: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:04.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: Reconfiguring mds.foofs.smithi069.nsgdqs (monmap changed)... 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: Reconfiguring daemon mds.foofs.smithi069.nsgdqs on smithi069 2024-06-11T14:00:06.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:06 smithi069 ceph-mon[66891]: pgmap v19: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: Reconfiguring mds.foofs.smithi069.nsgdqs (monmap changed)... 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: Reconfiguring daemon mds.foofs.smithi069.nsgdqs on smithi069 2024-06-11T14:00:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:06 smithi135 ceph-mon[53782]: pgmap v19: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:07 smithi069 ceph-mon[66891]: Reconfiguring crash.smithi135 (monmap changed)... 2024-06-11T14:00:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T14:00:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:07.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:07 smithi069 ceph-mon[66891]: Reconfiguring daemon crash.smithi135 on smithi135 2024-06-11T14:00:08.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:08.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:08.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:07 smithi135 ceph-mon[53782]: Reconfiguring crash.smithi135 (monmap changed)... 2024-06-11T14:00:08.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T14:00:08.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:08.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:07 smithi135 ceph-mon[53782]: Reconfiguring daemon crash.smithi135 on smithi135 2024-06-11T14:00:08.753 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (103s) 29s ago 23m 16.5M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (23m) 29s ago 23m 6405k - 17.2.0 e1d6a67b021e 40054c8248da 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (23m) 17s ago 23m 6405k - 17.2.0 e1d6a67b021e fb67de240762 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (99s) 29s ago 23m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (17m) 29s ago 17m 39.5M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (17m) 17s ago 17m 18.9M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (87s) 29s ago 25m 472M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (56s) 17s ago 23m 437M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:00:09.400 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (43s) 29s ago 25m 30.9M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (21s) 17s ago 22m 21.2M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (17m) 29s ago 17m 283M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (17m) 17s ago 17m 87.3M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (97s) 29s ago 23m 8702k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (91s) 17s ago 22m 16.9M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (22m) 29s ago 22m 499M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (21m) 29s ago 21m 412M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (21m) 29s ago 21m 528M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (20m) 29s ago 20m 414M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (20m) 17s ago 20m 459M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (20m) 17s ago 20m 533M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (19m) 17s ago 19m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (19m) 17s ago 19m 688M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:00:09.401 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (60s) 29s ago 22m 55.5M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:00:09.453 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:09 smithi135 ceph-mon[53782]: pgmap v20: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:09.453 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:09 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:09.454 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:09 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:09.454 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:09 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T14:00:09.454 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:09 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T14:00:09.454 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:09 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:09.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:09 smithi069 ceph-mon[66891]: pgmap v20: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:09 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:09 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:09 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi135.xcould", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-06-11T14:00:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:09 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mgr services"}]: dispatch 2024-06-11T14:00:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:09 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T14:00:09.799 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 4 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:00:09.800 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:00:10.133 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:00:10.133 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:00:10.133 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "4/23 daemons upgraded", 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mon daemons", 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:00:10.134 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:00:10.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: Reconfiguring mgr.smithi135.xcould (monmap changed)... 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: Reconfiguring daemon mgr.smithi135.xcould on smithi135 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:10 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3922720735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:10.530 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:00:10.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: Reconfiguring mgr.smithi135.xcould (monmap changed)... 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: Reconfiguring daemon mgr.smithi135.xcould on smithi135 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:10 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3922720735' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:11.344 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:11.344 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: Reconfiguring mon.smithi135 (monmap changed)... 2024-06-11T14:00:11.344 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: Reconfiguring daemon mon.smithi135 on smithi135 2024-06-11T14:00:11.344 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: pgmap v21: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:11.345 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:11.345 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1542105408' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:00:11.345 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:11.345 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:11.345 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-11T14:00:11.345 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: Reconfiguring mon.smithi135 (monmap changed)... 2024-06-11T14:00:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: Reconfiguring daemon mon.smithi135 on smithi135 2024-06-11T14:00:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: pgmap v21: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1542105408' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:00:11.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:11.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:11.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-11T14:00:11.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:12.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:12 smithi069 ceph-mon[66891]: Reconfiguring osd.4 (monmap changed)... 2024-06-11T14:00:12.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:12 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.4 on smithi135 2024-06-11T14:00:12.509 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:12 smithi135 ceph-mon[53782]: Reconfiguring osd.4 (monmap changed)... 2024-06-11T14:00:12.509 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:12 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.4 on smithi135 2024-06-11T14:00:13.435 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: pgmap v22: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:13.435 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:13.435 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:13.435 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: Reconfiguring osd.5 (monmap changed)... 2024-06-11T14:00:13.435 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-11T14:00:13.435 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:13.436 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:13 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.5 on smithi135 2024-06-11T14:00:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: pgmap v22: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:13.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: Reconfiguring osd.5 (monmap changed)... 2024-06-11T14:00:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-11T14:00:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:13 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.5 on smithi135 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: Reconfiguring osd.6 (monmap changed)... 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.6 on smithi135 2024-06-11T14:00:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:14 smithi135 ceph-mon[53782]: pgmap v23: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: Reconfiguring osd.6 (monmap changed)... 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.6 on smithi135 2024-06-11T14:00:15.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:14 smithi069 ceph-mon[66891]: pgmap v23: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: Reconfiguring osd.7 (monmap changed)... 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: Reconfiguring daemon osd.7 on smithi135 2024-06-11T14:00:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:16 smithi069 ceph-mon[66891]: pgmap v24: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:16.513 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:16.513 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:16.513 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: Reconfiguring osd.7 (monmap changed)... 2024-06-11T14:00:16.513 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-11T14:00:16.514 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:16.514 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: Reconfiguring daemon osd.7 on smithi135 2024-06-11T14:00:16.514 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:16 smithi135 ceph-mon[53782]: pgmap v24: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:17.895 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:17.895 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:17.895 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:17 smithi135 ceph-mon[53782]: Reconfiguring mds.foofs.smithi135.wtrawd (monmap changed)... 2024-06-11T14:00:17.895 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:00:17.896 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:17.896 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:17 smithi135 ceph-mon[53782]: Reconfiguring daemon mds.foofs.smithi135.wtrawd on smithi135 2024-06-11T14:00:17.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:17.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:17 smithi069 ceph-mon[66891]: Reconfiguring mds.foofs.smithi135.wtrawd (monmap changed)... 2024-06-11T14:00:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:00:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:17.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:17 smithi069 ceph-mon[66891]: Reconfiguring daemon mds.foofs.smithi135.wtrawd on smithi135 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all mon 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:19.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]: dispatch 2024-06-11T14:00:19.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]': finished 2024-06-11T14:00:19.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi135"}]: dispatch 2024-06-11T14:00:19.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi135"}]': finished 2024-06-11T14:00:19.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:18 smithi069 ceph-mon[66891]: pgmap v25: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all mon 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]: dispatch 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi069"}]': finished 2024-06-11T14:00:19.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi135"}]: dispatch 2024-06-11T14:00:19.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi135"}]': finished 2024-06-11T14:00:19.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:18 smithi135 ceph-mon[53782]: pgmap v25: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:20.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:20.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:20.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T14:00:20.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:20.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:20.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:20.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T14:00:20.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:21.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:21 smithi135 ceph-mon[53782]: Upgrade: Updating crash.smithi069 (1/2) 2024-06-11T14:00:21.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:21 smithi135 ceph-mon[53782]: Deploying daemon crash.smithi069 on smithi069 2024-06-11T14:00:21.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:21 smithi135 ceph-mon[53782]: pgmap v26: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:21.491 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:21 smithi069 ceph-mon[66891]: Upgrade: Updating crash.smithi069 (1/2) 2024-06-11T14:00:21.492 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:21 smithi069 ceph-mon[66891]: Deploying daemon crash.smithi069 on smithi069 2024-06-11T14:00:21.492 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:21 smithi069 ceph-mon[66891]: pgmap v26: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:23.178 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:23 smithi135 ceph-mon[53782]: pgmap v27: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:23.466 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:23 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:23.466 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:23 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:23 smithi069 ceph-mon[66891]: pgmap v27: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:23.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:23 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:23 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:25.391 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:25 smithi135 ceph-mon[53782]: pgmap v28: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:25.392 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:25 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:25.392 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:25 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T14:00:25.392 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:25 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:25 smithi069 ceph-mon[66891]: pgmap v28: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:25 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:25 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi135", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-06-11T14:00:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:25 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:26.222 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:26 smithi135 ceph-mon[53782]: Upgrade: Updating crash.smithi135 (2/2) 2024-06-11T14:00:26.222 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:26 smithi135 ceph-mon[53782]: Deploying daemon crash.smithi135 on smithi135 2024-06-11T14:00:26.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:26 smithi069 ceph-mon[66891]: Upgrade: Updating crash.smithi135 (2/2) 2024-06-11T14:00:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:26 smithi069 ceph-mon[66891]: Deploying daemon crash.smithi135 on smithi135 2024-06-11T14:00:27.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:27 smithi135 ceph-mon[53782]: pgmap v29: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:27.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:27 smithi069 ceph-mon[66891]: pgmap v29: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:28.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:28.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:28.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:28.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:28 smithi135 ceph-mon[53782]: pgmap v30: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:28.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:28.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:28.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:28.969 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:28 smithi069 ceph-mon[66891]: pgmap v30: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:31.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:31 smithi069 ceph-mon[66891]: pgmap v31: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:31.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:31 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:31.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:31 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:31 smithi135 ceph-mon[53782]: pgmap v31: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:31 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:31 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:32.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:32.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:32.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:32 smithi069 ceph-mon[66891]: pgmap v32: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:32.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:32 smithi135 ceph-mon[53782]: pgmap v32: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:34.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.202 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.203 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:34.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:34 smithi135 ceph-mon[53782]: pgmap v33: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:34 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:34 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:35.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:34 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:35.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:34 smithi069 ceph-mon[66891]: pgmap v33: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:35.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:35.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:35.169 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: pgmap v34: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:37.387 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]: dispatch 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]': finished 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi135"}]: dispatch 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi135"}]': finished 2024-06-11T14:00:37.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-11T14:00:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: pgmap v34: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi069"}]': finished 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi135"}]: dispatch 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi135"}]': finished 2024-06-11T14:00:37.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-11T14:00:38.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:38 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all crash 2024-06-11T14:00:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:38 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-11T14:00:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:38 smithi069 ceph-mon[66891]: Upgrade: osd.0 is safe to restart 2024-06-11T14:00:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:38 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all crash 2024-06-11T14:00:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:38 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-06-11T14:00:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:38 smithi135 ceph-mon[53782]: Upgrade: osd.0 is safe to restart 2024-06-11T14:00:39.456 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:39 smithi135 ceph-mon[53782]: pgmap v35: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:39.457 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:39 smithi135 ceph-mon[53782]: Upgrade: Updating osd.0 2024-06-11T14:00:39.457 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:39 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:39.457 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:39 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-11T14:00:39.457 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:39 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:39.457 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:39 smithi135 ceph-mon[53782]: Deploying daemon osd.0 on smithi069 2024-06-11T14:00:39.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:39 smithi069 ceph-mon[66891]: pgmap v35: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:39.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:39 smithi069 ceph-mon[66891]: Upgrade: Updating osd.0 2024-06-11T14:00:39.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:39 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:39.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:39 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-06-11T14:00:39.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:39 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:39.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:39 smithi069 ceph-mon[66891]: Deploying daemon osd.0 on smithi069 2024-06-11T14:00:40.899 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:00:41.448 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:41 smithi135 ceph-mon[53782]: pgmap v36: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:41.458 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:41 smithi069 ceph-mon[66891]: pgmap v36: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:00:41.458 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:41 smithi069 systemd[1]: Stopping Ceph osd.0 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (2m) 8s ago 24m 16.9M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (19s) 8s ago 24m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (14s) 10s ago 23m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (2m) 8s ago 24m 43.2M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (18m) 8s ago 18m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (18m) 10s ago 18m 19.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:00:41.603 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (119s) 8s ago 25m 492M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (88s) 10s ago 23m 438M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (75s) 8s ago 25m 43.2M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (54s) 10s ago 23m 34.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (18m) 8s ago 18m 294M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (18m) 10s ago 18m 87.4M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (2m) 8s ago 23m 8590k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (2m) 10s ago 23m 16.3M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (22m) 8s ago 22m 500M 1936M 17.2.0 e1d6a67b021e 0769173bf341 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (22m) 8s ago 22m 411M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (21m) 8s ago 21m 529M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (21m) 8s ago 21m 414M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (21m) 10s ago 21m 459M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T14:00:41.604 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (20m) 10s ago 20m 534M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:00:41.605 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (20m) 10s ago 20m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:00:41.605 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (19m) 10s ago 19m 689M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:00:41.605 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (93s) 8s ago 23m 64.7M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:00:41.747 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:41 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[35117]: 2024-06-11T14:00:41.457+0000 7f191351b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:00:41.748 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:41 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[35117]: 2024-06-11T14:00:41.457+0000 7f191351b700 -1 osd.0 84 *** Got signal Terminated *** 2024-06-11T14:00:41.748 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:41 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[35117]: 2024-06-11T14:00:41.457+0000 7f191351b700 -1 osd.0 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-06-11T14:00:41.998 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 4 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:00:41.999 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:00:42.341 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:00:42.342 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:00:42.342 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "6/23 daemons upgraded", 2024-06-11T14:00:42.342 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:00:42.342 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:00:42.342 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:00:42.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:42 smithi069 ceph-mon[66891]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:42.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:42 smithi069 ceph-mon[66891]: osd.0 marked itself down and dead 2024-06-11T14:00:42.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:42 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2158620508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:42 smithi135 ceph-mon[53782]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:42 smithi135 ceph-mon[53782]: osd.0 marked itself down and dead 2024-06-11T14:00:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:42 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2158620508' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:42.739 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN 1 osds down 2024-06-11T14:00:42.739 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] OSD_DOWN: 1 osds down 2024-06-11T14:00:42.739 INFO:teuthology.orchestra.run.smithi069.stdout: osd.0 (root=default,host=smithi069) is down 2024-06-11T14:00:42.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:42 smithi069 podman[72343]: 2024-06-11 14:00:42.770841096 +0000 UTC m=+1.428419092 container died 0769173bf341e2482ff89c5ef02910cf3bd49f59577be528cccfba2fb1039e9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, name=centos-stream, GIT_CLEAN=True, version=8, distribution-scope=public, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-06-11T14:00:43.409 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:43 smithi069 podman[72343]: 2024-06-11 14:00:43.150109121 +0000 UTC m=+1.807687116 container cleanup 0769173bf341e2482ff89c5ef02910cf3bd49f59577be528cccfba2fb1039e9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, ceph=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, release=754) 2024-06-11T14:00:43.409 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:43 smithi069 bash[72343]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0 2024-06-11T14:00:43.409 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:43 smithi069 ceph-mon[66891]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:43.409 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:43 smithi069 ceph-mon[66891]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:43.409 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:43 smithi069 ceph-mon[66891]: pgmap v37: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:43.409 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:43 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:00:43.409 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:43 smithi069 ceph-mon[66891]: osdmap e85: 8 total, 7 up, 8 in 2024-06-11T14:00:43.410 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:43 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/4104175413' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:00:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:43 smithi135 ceph-mon[53782]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:43 smithi135 ceph-mon[53782]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:43 smithi135 ceph-mon[53782]: pgmap v37: 97 pgs: 97 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:00:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:43 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:00:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:43 smithi135 ceph-mon[53782]: osdmap e85: 8 total, 7 up, 8 in 2024-06-11T14:00:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:43 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/4104175413' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:00:43.728 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:43 smithi069 podman[72452]: 2024-06-11 14:00:43.410024012 +0000 UTC m=+0.642033242 container remove 0769173bf341e2482ff89c5ef02910cf3bd49f59577be528cccfba2fb1039e9e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, RELEASE=HEAD, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-type=git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-06-11T14:00:43.997 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:43 smithi069 podman[72570]: 2024-06-11 14:00:43.632279902 +0000 UTC m=+0.017302871 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:00:43.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:43 smithi069 podman[72570]: 2024-06-11 14:00:43.835910829 +0000 UTC m=+0.220933824 container create 24788077408bc3b4f0ea2b3e6e3c7b58c6f38188fab7f57e1c53e3dc534e3323 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0) 2024-06-11T14:00:44.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:44 smithi069 ceph-mon[66891]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:44.317 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:44 smithi069 ceph-mon[66891]: osdmap e86: 8 total, 7 up, 8 in 2024-06-11T14:00:44.317 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 podman[72570]: 2024-06-11 14:00:44.055066276 +0000 UTC m=+0.440089242 container init 24788077408bc3b4f0ea2b3e6e3c7b58c6f38188fab7f57e1c53e3dc534e3323 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-06-11T14:00:44.317 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 podman[72570]: 2024-06-11 14:00:44.058945109 +0000 UTC m=+0.443968070 container start 24788077408bc3b4f0ea2b3e6e3c7b58c6f38188fab7f57e1c53e3dc534e3323 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, ceph=True, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:00:44.317 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 podman[72570]: 2024-06-11 14:00:44.146787441 +0000 UTC m=+0.531810409 container attach 24788077408bc3b4f0ea2b3e6e3c7b58c6f38188fab7f57e1c53e3dc534e3323 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-06-11T14:00:44.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:44 smithi135 ceph-mon[53782]: from='client.34196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:00:44.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:44 smithi135 ceph-mon[53782]: osdmap e86: 8 total, 7 up, 8 in 2024-06-11T14:00:44.599 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 podman[72570]: 2024-06-11 14:00:44.318874033 +0000 UTC m=+0.703896999 container died 24788077408bc3b4f0ea2b3e6e3c7b58c6f38188fab7f57e1c53e3dc534e3323 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-06-11T14:00:45.186 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 podman[72595]: 2024-06-11 14:00:44.93370495 +0000 UTC m=+0.609187443 container remove 24788077408bc3b4f0ea2b3e6e3c7b58c6f38188fab7f57e1c53e3dc534e3323 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:00:45.186 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.0.service: Deactivated successfully. 2024-06-11T14:00:45.186 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 systemd[1]: Stopped Ceph osd.0 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:00:45.186 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:44 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.0.service: Consumed 4min 10.052s CPU time. 2024-06-11T14:00:45.186 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:45 smithi069 systemd[1]: Starting Ceph osd.0 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:00:45.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:45 smithi069 ceph-mon[66891]: pgmap v40: 97 pgs: 13 stale+active+clean, 84 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:00:45.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:45 smithi069 podman[72696]: 2024-06-11 14:00:45.295547024 +0000 UTC m=+0.017397363 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:00:45.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:45 smithi069 podman[72696]: 2024-06-11 14:00:45.426614161 +0000 UTC m=+0.148464482 container create e5d99d5ec3918ef8e8fc5ef5817fd719a0dd7131da82ed10a4d2c1ba5386ce1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528) 2024-06-11T14:00:45.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:45 smithi135 ceph-mon[53782]: pgmap v40: 97 pgs: 13 stale+active+clean, 84 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:00:45.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:45 smithi069 podman[72696]: 2024-06-11 14:00:45.65420497 +0000 UTC m=+0.376055311 container init e5d99d5ec3918ef8e8fc5ef5817fd719a0dd7131da82ed10a4d2c1ba5386ce1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3) 2024-06-11T14:00:45.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:45 smithi069 podman[72696]: 2024-06-11 14:00:45.657704451 +0000 UTC m=+0.379554779 container start e5d99d5ec3918ef8e8fc5ef5817fd719a0dd7131da82ed10a4d2c1ba5386ce1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:00:45.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:45 smithi069 podman[72696]: 2024-06-11 14:00:45.754039563 +0000 UTC m=+0.475889896 container attach e5d99d5ec3918ef8e8fc5ef5817fd719a0dd7131da82ed10a4d2c1ba5386ce1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:00:46.381 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-mon[66891]: pgmap v41: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-11T14:00:46.382 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-06-11T14:00:46.383 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-11T14:00:46.383 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-06-11T14:00:46.383 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate[72710]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-11T14:00:46.383 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 bash[72696]: --> ceph-volume raw activate successful for osd ID: 0 2024-06-11T14:00:46.383 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 podman[72696]: 2024-06-11 14:00:46.072163989 +0000 UTC m=+0.794014317 container died e5d99d5ec3918ef8e8fc5ef5817fd719a0dd7131da82ed10a4d2c1ba5386ce1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0) 2024-06-11T14:00:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:46 smithi135 ceph-mon[53782]: pgmap v41: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:46.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 podman[72833]: 2024-06-11 14:00:46.691327925 +0000 UTC m=+0.609578508 container remove e5d99d5ec3918ef8e8fc5ef5817fd719a0dd7131da82ed10a4d2c1ba5386ce1f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-06-11T14:00:46.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 podman[72892]: 2024-06-11 14:00:46.900420372 +0000 UTC m=+0.111850655 container create 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, ceph=True, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0) 2024-06-11T14:00:46.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:46 smithi069 podman[72892]: 2024-06-11 14:00:46.805445452 +0000 UTC m=+0.016875754 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:00:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:47 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 69/426 objects degraded (16.197%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T14:00:47.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:47 smithi069 podman[72892]: 2024-06-11 14:00:47.127970605 +0000 UTC m=+0.339400906 container init 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-06-11T14:00:47.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:47 smithi069 podman[72892]: 2024-06-11 14:00:47.13124241 +0000 UTC m=+0.342672707 container start 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528) 2024-06-11T14:00:47.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:47 smithi069 ceph-osd[72911]: -- 172.21.15.69:0/504936618 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c613d461e0 con 0x55c612f41c00 2024-06-11T14:00:47.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:47 smithi069 bash[72892]: 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 2024-06-11T14:00:47.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:47 smithi069 systemd[1]: Started Ceph osd.0 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:00:47.546 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:47 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 69/426 objects degraded (16.197%), 18 pgs degraded (PG_DEGRADED) 2024-06-11T14:00:47.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:47 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:00:47.669+0000 7f4b0843f640 -1 Falling back to public interface 2024-06-11T14:00:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:48 smithi135 ceph-mon[53782]: pgmap v42: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 69/426 objects degraded (16.197%) 2024-06-11T14:00:48.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:48.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:48.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:48 smithi069 ceph-mon[66891]: pgmap v42: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 69/426 objects degraded (16.197%) 2024-06-11T14:00:50.388 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:50 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:50.389 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:50 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:50 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:50 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:00:51.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:51 smithi069 ceph-mon[66891]: pgmap v43: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:51 smithi135 ceph-mon[53782]: pgmap v43: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:53.498 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:53 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:00:53.361+0000 7f4b0843f640 -1 osd.0 0 read_superblock omap replica is missing. 2024-06-11T14:00:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:53 smithi069 ceph-mon[66891]: pgmap v44: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 206 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:53 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:53 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:53 smithi135 ceph-mon[53782]: pgmap v44: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 206 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:53 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:53 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:54.637 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:54 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:00:54.443+0000 7f4b0843f640 -1 osd.0 84 log_to_monitors true 2024-06-11T14:00:54.997 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:54 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:54.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:54 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:54.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:54 smithi069 ceph-mon[66891]: pgmap v45: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 187 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:54.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:54 smithi069 ceph-mon[66891]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-11T14:00:55.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:54 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:55.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:54 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:55.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:54 smithi135 ceph-mon[53782]: pgmap v45: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 187 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:55.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:54 smithi135 ceph-mon[53782]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-06-11T14:00:55.428 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:00:55 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:00:55.124+0000 7f4afc1e2640 -1 osd.0 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:00:55.747 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:55 smithi069 ceph-mon[66891]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-11T14:00:55.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:55 smithi069 ceph-mon[66891]: osdmap e87: 8 total, 7 up, 8 in 2024-06-11T14:00:55.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:55 smithi069 ceph-mon[66891]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:00:55.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:55 smithi069 ceph-mon[66891]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' 2024-06-11T14:00:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:55 smithi135 ceph-mon[53782]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-06-11T14:00:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:55 smithi135 ceph-mon[53782]: osdmap e87: 8 total, 7 up, 8 in 2024-06-11T14:00:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:55 smithi135 ceph-mon[53782]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:00:56.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:55 smithi135 ceph-mon[53782]: from='osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421]' entity='osd.0' 2024-06-11T14:00:56.934 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:56 smithi069 ceph-mon[66891]: pgmap v47: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:56.934 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:56 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:00:56.934 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:56 smithi069 ceph-mon[66891]: osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421] boot 2024-06-11T14:00:56.934 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:56 smithi069 ceph-mon[66891]: osdmap e88: 8 total, 8 up, 8 in 2024-06-11T14:00:56.934 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T14:00:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:56 smithi135 ceph-mon[53782]: pgmap v47: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 69/426 objects degraded (16.197%) 2024-06-11T14:00:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:56 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:00:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:56 smithi135 ceph-mon[53782]: osd.0 [v2:172.21.15.69:6802/3431422421,v1:172.21.15.69:6803/3431422421] boot 2024-06-11T14:00:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:56 smithi135 ceph-mon[53782]: osdmap e88: 8 total, 8 up, 8 in 2024-06-11T14:00:57.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-06-11T14:00:58.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:57 smithi069 ceph-mon[66891]: osdmap e89: 8 total, 8 up, 8 in 2024-06-11T14:00:58.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:00:58.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:00:58.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-06-11T14:00:58.299 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:57 smithi135 ceph-mon[53782]: osdmap e89: 8 total, 8 up, 8 in 2024-06-11T14:00:59.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:00:58 smithi069 ceph-mon[66891]: pgmap v50: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 69/426 objects degraded (16.197%) 2024-06-11T14:00:59.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:00:58 smithi135 ceph-mon[53782]: pgmap v50: 97 pgs: 18 active+undersized, 18 active+undersized+degraded, 61 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 69/426 objects degraded (16.197%) 2024-06-11T14:01:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:00 smithi069 ceph-mon[66891]: pgmap v51: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:01:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:00 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/426 objects degraded (16.197%), 18 pgs degraded) 2024-06-11T14:01:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:00 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:01:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:00 smithi135 ceph-mon[53782]: pgmap v51: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:01:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:00 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/426 objects degraded (16.197%), 18 pgs degraded) 2024-06-11T14:01:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:00 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:01:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:03 smithi069 ceph-mon[66891]: pgmap v52: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 275 B/s rd, 0 op/s 2024-06-11T14:01:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:03 smithi135 ceph-mon[53782]: pgmap v52: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 275 B/s rd, 0 op/s 2024-06-11T14:01:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:05 smithi069 ceph-mon[66891]: pgmap v53: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:01:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:05 smithi135 ceph-mon[53782]: pgmap v53: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:01:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:07 smithi069 ceph-mon[66891]: pgmap v54: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 205 B/s rd, 308 B/s wr, 0 op/s 2024-06-11T14:01:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:07 smithi069 ceph-mon[66891]: mgrmap e40: smithi069.iruzyw(active, since 92s), standbys: smithi135.xcould 2024-06-11T14:01:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:07 smithi135 ceph-mon[53782]: pgmap v54: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 205 B/s rd, 308 B/s wr, 0 op/s 2024-06-11T14:01:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:07 smithi135 ceph-mon[53782]: mgrmap e40: smithi069.iruzyw(active, since 92s), standbys: smithi135.xcould 2024-06-11T14:01:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:09 smithi069 ceph-mon[66891]: pgmap v55: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 186 B/s rd, 280 B/s wr, 0 op/s 2024-06-11T14:01:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:09 smithi135 ceph-mon[53782]: pgmap v55: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 186 B/s rd, 280 B/s wr, 0 op/s 2024-06-11T14:01:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:11.448 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:11 smithi135 ceph-mon[53782]: pgmap v56: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:11.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:11 smithi069 ceph-mon[66891]: pgmap v56: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:12.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:01:12.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:01:13.097 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:01:13.414 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:13 smithi069 ceph-mon[66891]: pgmap v57: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:13.414 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:13 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:01:13.414 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:13 smithi069 ceph-mon[66891]: Upgrade: osd.1 is safe to restart 2024-06-11T14:01:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:13 smithi135 ceph-mon[53782]: pgmap v57: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:13 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-06-11T14:01:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:13 smithi135 ceph-mon[53782]: Upgrade: osd.1 is safe to restart 2024-06-11T14:01:13.778 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:01:13.778 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (2m) 20s ago 24m 17.2M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:01:13.778 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (51s) 20s ago 24m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (46s) 42s ago 24m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (2m) 20s ago 24m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (19m) 20s ago 19m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (18m) 42s ago 18m 19.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (2m) 20s ago 26m 497M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (2m) 42s ago 24m 438M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (108s) 20s ago 26m 44.3M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (86s) 42s ago 24m 34.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (18m) 20s ago 18m 294M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (18m) 42s ago 18m 87.4M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (2m) 20s ago 24m 9000k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (2m) 42s ago 23m 16.3M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (26s) 20s ago 23m 12.0M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:01:13.779 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (22m) 20s ago 22m 412M 1936M 17.2.0 e1d6a67b021e 51d3db4a7ba3 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (22m) 20s ago 22m 528M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (22m) 20s ago 22m 413M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (21m) 42s ago 21m 459M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (21m) 42s ago 21m 534M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (20m) 42s ago 20m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (20m) 42s ago 20m 689M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:01:13.780 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (2m) 20s ago 23m 64.9M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:01:14.193 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:01:14.193 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:01:14.193 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:01:14.193 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:14.193 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:01:14.193 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 1 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 5 2024-06-11T14:01:14.194 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:01:14.195 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:01:14.494 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:14 smithi069 ceph-mon[66891]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:14.494 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:14.495 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-11T14:01:14.495 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:01:14.529 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "7/23 daemons upgraded", 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:01:14.530 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:01:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:14 smithi135 ceph-mon[53782]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-06-11T14:01:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:01:14.932 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: Upgrade: Updating osd.1 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: Deploying daemon osd.1 on smithi069 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: pgmap v58: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3082596439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:15.394 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:15 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/426262395' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:01:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: Upgrade: Updating osd.1 2024-06-11T14:01:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: Deploying daemon osd.1 on smithi069 2024-06-11T14:01:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: pgmap v58: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3082596439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:15 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/426262395' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:01:16.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:16 smithi069 ceph-mon[66891]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:16.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:16 smithi135 ceph-mon[53782]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:16.608 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:16 smithi069 systemd[1]: Stopping Ceph osd.1 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:01:16.998 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:16 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T14:01:16.608+0000 7fa8445e4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:01:16.998 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:16 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T14:01:16.608+0000 7fa8445e4700 -1 osd.1 89 *** Got signal Terminated *** 2024-06-11T14:01:16.998 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:16 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[38223]: 2024-06-11T14:01:16.608+0000 7fa8445e4700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:01:17.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:17 smithi069 ceph-mon[66891]: pgmap v59: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:17 smithi069 ceph-mon[66891]: osd.1 marked itself down and dead 2024-06-11T14:01:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:17 smithi135 ceph-mon[53782]: pgmap v59: 97 pgs: 97 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 255 B/s wr, 0 op/s 2024-06-11T14:01:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:17 smithi135 ceph-mon[53782]: osd.1 marked itself down and dead 2024-06-11T14:01:17.998 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:17 smithi069 podman[75068]: 2024-06-11 14:01:17.763594354 +0000 UTC m=+1.277953003 container died 51d3db4a7ba37ef05dbc358acf7c8cf3cda8937198c6d758b008ef5ac1decb08 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , version=8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-06-11T14:01:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:18 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:01:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:18 smithi069 ceph-mon[66891]: osdmap e90: 8 total, 7 up, 8 in 2024-06-11T14:01:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:18 smithi069 ceph-mon[66891]: osdmap e91: 8 total, 7 up, 8 in 2024-06-11T14:01:18.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:18 smithi069 podman[75068]: 2024-06-11 14:01:18.276108679 +0000 UTC m=+1.790467335 container cleanup 51d3db4a7ba37ef05dbc358acf7c8cf3cda8937198c6d758b008ef5ac1decb08 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, name=centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, release=754, ceph=True, RELEASE=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-11T14:01:18.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:18 smithi069 bash[75068]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1 2024-06-11T14:01:18.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:18 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:01:18.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:18 smithi135 ceph-mon[53782]: osdmap e90: 8 total, 7 up, 8 in 2024-06-11T14:01:18.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:18 smithi135 ceph-mon[53782]: osdmap e91: 8 total, 7 up, 8 in 2024-06-11T14:01:18.892 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:18 smithi069 podman[75094]: 2024-06-11 14:01:18.561086058 +0000 UTC m=+0.793528136 container remove 51d3db4a7ba37ef05dbc358acf7c8cf3cda8937198c6d758b008ef5ac1decb08 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public) 2024-06-11T14:01:19.173 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:18 smithi069 podman[75185]: 2024-06-11 14:01:18.79582271 +0000 UTC m=+0.016898203 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:01:19.174 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:18 smithi069 podman[75185]: 2024-06-11 14:01:18.92008128 +0000 UTC m=+0.141156762 container create 51eec55b8f7ebdd173cd30d6cd55aaac1e32dc8138d837567aecfc5539cdfb47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, org.label-schema.build-date=20240528, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-06-11T14:01:19.174 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 podman[75185]: 2024-06-11 14:01:19.114700547 +0000 UTC m=+0.335776032 container init 51eec55b8f7ebdd173cd30d6cd55aaac1e32dc8138d837567aecfc5539cdfb47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-11T14:01:19.174 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 podman[75185]: 2024-06-11 14:01:19.118230755 +0000 UTC m=+0.339306232 container start 51eec55b8f7ebdd173cd30d6cd55aaac1e32dc8138d837567aecfc5539cdfb47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=) 2024-06-11T14:01:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:19 smithi069 ceph-mon[66891]: pgmap v61: 97 pgs: 14 stale+active+clean, 83 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:01:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:19.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 podman[75185]: 2024-06-11 14:01:19.206154532 +0000 UTC m=+0.427230025 container attach 51eec55b8f7ebdd173cd30d6cd55aaac1e32dc8138d837567aecfc5539cdfb47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:01:19.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 podman[75185]: 2024-06-11 14:01:19.375436596 +0000 UTC m=+0.596512078 container died 51eec55b8f7ebdd173cd30d6cd55aaac1e32dc8138d837567aecfc5539cdfb47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:01:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:19 smithi135 ceph-mon[53782]: pgmap v61: 97 pgs: 14 stale+active+clean, 83 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:01:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:19.944 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 podman[75209]: 2024-06-11 14:01:19.926634107 +0000 UTC m=+0.545925064 container remove 51eec55b8f7ebdd173cd30d6cd55aaac1e32dc8138d837567aecfc5539cdfb47 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:01:20.225 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.1.service: Deactivated successfully. 2024-06-11T14:01:20.225 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 systemd[1]: Stopped Ceph osd.1 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:01:20.225 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:19 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.1.service: Consumed 4min 5.110s CPU time. 2024-06-11T14:01:20.226 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 systemd[1]: Starting Ceph osd.1 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:01:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:20 smithi069 ceph-mon[66891]: pgmap v63: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:20 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 49/426 objects degraded (11.502%), 12 pgs degraded (PG_DEGRADED) 2024-06-11T14:01:20.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 podman[75307]: 2024-06-11 14:01:20.274707004 +0000 UTC m=+0.017528517 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:01:20.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 podman[75307]: 2024-06-11 14:01:20.377463791 +0000 UTC m=+0.120285298 container create 155c505e8f33d1272967f5d6ee42e78801740ffd7c7e76264699493be5c5ece8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, GIT_CLEAN=True, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-06-11T14:01:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:20 smithi135 ceph-mon[53782]: pgmap v63: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:20 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 49/426 objects degraded (11.502%), 12 pgs degraded (PG_DEGRADED) 2024-06-11T14:01:20.968 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 podman[75307]: 2024-06-11 14:01:20.605438622 +0000 UTC m=+0.348260142 container init 155c505e8f33d1272967f5d6ee42e78801740ffd7c7e76264699493be5c5ece8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate, org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-06-11T14:01:20.969 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 podman[75307]: 2024-06-11 14:01:20.608945199 +0000 UTC m=+0.351766718 container start 155c505e8f33d1272967f5d6ee42e78801740ffd7c7e76264699493be5c5ece8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, ceph=True, RELEASE=squid-3f51ea3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS) 2024-06-11T14:01:20.969 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 podman[75307]: 2024-06-11 14:01:20.67992549 +0000 UTC m=+0.422747015 container attach 155c505e8f33d1272967f5d6ee42e78801740ffd7c7e76264699493be5c5ece8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3) 2024-06-11T14:01:21.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-11T14:01:21.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 bash[75307]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-11T14:01:21.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-11T14:01:21.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 bash[75307]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-11T14:01:21.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:20 smithi069 bash[75307]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 bash[75307]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 bash[75307]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 bash[75307]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate[75323]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 bash[75307]: --> ceph-volume raw activate successful for osd ID: 1 2024-06-11T14:01:21.249 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 podman[75307]: 2024-06-11 14:01:21.030314907 +0000 UTC m=+0.773136417 container died 155c505e8f33d1272967f5d6ee42e78801740ffd7c7e76264699493be5c5ece8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3) 2024-06-11T14:01:21.748 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 podman[75442]: 2024-06-11 14:01:21.633887964 +0000 UTC m=+0.597821113 container remove 155c505e8f33d1272967f5d6ee42e78801740ffd7c7e76264699493be5c5ece8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-activate, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-06-11T14:01:22.186 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 podman[75504]: 2024-06-11 14:01:21.750066665 +0000 UTC m=+0.017313446 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:01:22.186 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:21 smithi069 podman[75504]: 2024-06-11 14:01:21.851267368 +0000 UTC m=+0.118514136 container create 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3) 2024-06-11T14:01:22.187 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:22 smithi069 podman[75504]: 2024-06-11 14:01:22.070257643 +0000 UTC m=+0.337504417 container init 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-06-11T14:01:22.187 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:22 smithi069 podman[75504]: 2024-06-11 14:01:22.073539945 +0000 UTC m=+0.340786714 container start 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, GIT_BRANCH=HEAD) 2024-06-11T14:01:22.187 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:22 smithi069 ceph-osd[75523]: -- 172.21.15.69:0/38585677 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5597364141e0 con 0x55973560fc00 2024-06-11T14:01:22.187 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:22 smithi069 bash[75504]: 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 2024-06-11T14:01:22.497 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:22 smithi069 systemd[1]: Started Ceph osd.1 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:01:22.913 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:22 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:01:22.613+0000 7f2d95333640 -1 Falling back to public interface 2024-06-11T14:01:23.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:23 smithi069 ceph-mon[66891]: pgmap v64: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:23.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:23 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:23.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:23 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:23.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:23 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:01:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:23 smithi135 ceph-mon[53782]: pgmap v64: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:23 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:23 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:23 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:01:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:25 smithi069 ceph-mon[66891]: pgmap v65: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:25.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:25 smithi135 ceph-mon[53782]: pgmap v65: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:27.450 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:27 smithi069 ceph-mon[66891]: pgmap v66: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 229 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:27 smithi135 ceph-mon[53782]: pgmap v66: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 229 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:27.747 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:27 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:01:27.666+0000 7f2d95333640 -1 osd.1 0 read_superblock omap replica is missing. 2024-06-11T14:01:28.657 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:01:28.526+0000 7f2d95333640 -1 osd.1 89 log_to_monitors true 2024-06-11T14:01:28.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:28.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:28.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-mon[66891]: pgmap v67: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:28.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:28.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:28.985 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:28 smithi069 ceph-mon[66891]: from='osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-11T14:01:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:28 smithi135 ceph-mon[53782]: pgmap v67: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:28 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:29.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:28 smithi135 ceph-mon[53782]: from='osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-06-11T14:01:29.748 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:01:29 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:01:29.479+0000 7f2d8d0de640 -1 osd.1 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:01:30.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:30 smithi069 ceph-mon[66891]: from='osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-11T14:01:30.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:30 smithi069 ceph-mon[66891]: osdmap e92: 8 total, 7 up, 8 in 2024-06-11T14:01:30.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:30 smithi069 ceph-mon[66891]: from='osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:01:30.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:30 smithi069 ceph-mon[66891]: pgmap v69: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:30.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:30 smithi135 ceph-mon[53782]: from='osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-06-11T14:01:30.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:30 smithi135 ceph-mon[53782]: osdmap e92: 8 total, 7 up, 8 in 2024-06-11T14:01:30.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:30 smithi135 ceph-mon[53782]: from='osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:01:30.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:30 smithi135 ceph-mon[53782]: pgmap v69: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:31.734 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:31 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:01:31.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:31 smithi069 ceph-mon[66891]: osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683] boot 2024-06-11T14:01:31.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:31 smithi069 ceph-mon[66891]: osdmap e93: 8 total, 8 up, 8 in 2024-06-11T14:01:31.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:31 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T14:01:31.735 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:31 smithi069 ceph-mon[66891]: osdmap e94: 8 total, 8 up, 8 in 2024-06-11T14:01:31.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:31 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:01:31.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:31 smithi135 ceph-mon[53782]: osd.1 [v2:172.21.15.69:6810/313732683,v1:172.21.15.69:6811/313732683] boot 2024-06-11T14:01:31.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:31 smithi135 ceph-mon[53782]: osdmap e93: 8 total, 8 up, 8 in 2024-06-11T14:01:31.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:31 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-06-11T14:01:31.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:31 smithi135 ceph-mon[53782]: osdmap e94: 8 total, 8 up, 8 in 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-11T14:01:33.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:32 smithi135 ceph-mon[53782]: pgmap v72: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:33.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:01:33.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:01:33.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-06-11T14:01:33.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:32 smithi069 ceph-mon[66891]: pgmap v72: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:35 smithi069 ceph-mon[66891]: pgmap v73: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:35 smithi135 ceph-mon[53782]: pgmap v73: 97 pgs: 27 active+undersized, 12 active+undersized+degraded, 58 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 49/426 objects degraded (11.502%) 2024-06-11T14:01:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:36.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:36 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 49/426 objects degraded (11.502%), 12 pgs degraded) 2024-06-11T14:01:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:36 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:01:36.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:36 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 49/426 objects degraded (11.502%), 12 pgs degraded) 2024-06-11T14:01:36.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:36 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:01:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:37 smithi069 ceph-mon[66891]: pgmap v74: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 308 B/s rd, 0 op/s 2024-06-11T14:01:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:37 smithi135 ceph-mon[53782]: pgmap v74: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 308 B/s rd, 0 op/s 2024-06-11T14:01:39.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:39 smithi069 ceph-mon[66891]: pgmap v75: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-06-11T14:01:39.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:39 smithi135 ceph-mon[53782]: pgmap v75: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-06-11T14:01:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:41 smithi069 ceph-mon[66891]: pgmap v76: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 212 B/s rd, 0 op/s 2024-06-11T14:01:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:41 smithi135 ceph-mon[53782]: pgmap v76: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 212 B/s rd, 0 op/s 2024-06-11T14:01:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:43 smithi069 ceph-mon[66891]: pgmap v77: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 192 B/s rd, 0 op/s 2024-06-11T14:01:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:43 smithi135 ceph-mon[53782]: pgmap v77: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 192 B/s rd, 0 op/s 2024-06-11T14:01:45.303 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:01:45.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:45 smithi069 ceph-mon[66891]: pgmap v78: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:45.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:45 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:45.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:45 smithi135 ceph-mon[53782]: pgmap v78: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:45.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:45 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (3m) 18s ago 25m 17.2M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (83s) 18s ago 25m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (78s) 74s ago 24m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (3m) 18s ago 25m 43.4M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (19m) 18s ago 19m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (19m) 74s ago 19m 19.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (3m) 18s ago 26m 499M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (2m) 74s ago 24m 438M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (2m) 18s ago 26m 46.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (118s) 74s ago 24m 34.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:01:45.954 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (19m) 18s ago 19m 296M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (19m) 74s ago 19m 87.4M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (3m) 18s ago 25m 8502k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (3m) 74s ago 24m 16.3M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (59s) 18s ago 23m 206M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (24s) 18s ago 23m 11.9M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (22m) 18s ago 22m 529M 1936M 17.2.0 e1d6a67b021e fc965113df05 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (22m) 18s ago 22m 413M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (22m) 74s ago 22m 459M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (21m) 74s ago 21m 534M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (21m) 74s ago 21m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (20m) 74s ago 20m 689M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:01:45.955 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (2m) 18s ago 24m 74.0M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-11T14:01:46.349 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 6 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:01:46.350 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "8/23 daemons upgraded", 2024-06-11T14:01:46.697 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:01:46.698 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:01:46.698 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:01:47.102 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: pgmap v79: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2522677296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:47.428 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:47 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/4127659291' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:01:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: pgmap v79: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2522677296' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:01:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:47.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:47 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/4127659291' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:01:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:48 smithi069 ceph-mon[66891]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:48 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:48.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:48 smithi069 ceph-mon[66891]: Upgrade: osd.2 is safe to restart 2024-06-11T14:01:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:48 smithi135 ceph-mon[53782]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:01:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:48 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-06-11T14:01:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:48 smithi135 ceph-mon[53782]: Upgrade: osd.2 is safe to restart 2024-06-11T14:01:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:49 smithi069 ceph-mon[66891]: pgmap v80: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-11T14:01:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:01:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:49 smithi135 ceph-mon[53782]: pgmap v80: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-06-11T14:01:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:01:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:01:50.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:50 smithi069 ceph-mon[66891]: Upgrade: Updating osd.2 2024-06-11T14:01:50.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:50 smithi069 ceph-mon[66891]: Deploying daemon osd.2 on smithi069 2024-06-11T14:01:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:50 smithi135 ceph-mon[53782]: Upgrade: Updating osd.2 2024-06-11T14:01:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:50 smithi135 ceph-mon[53782]: Deploying daemon osd.2 on smithi069 2024-06-11T14:01:51.492 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:51 smithi069 ceph-mon[66891]: pgmap v81: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:01:51.493 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:51 smithi069 systemd[1]: Stopping Ceph osd.2 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:01:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:51 smithi135 ceph-mon[53782]: pgmap v81: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:01:51.747 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:51 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T14:01:51.492+0000 7fc30c1fa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:01:51.748 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:51 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T14:01:51.492+0000 7fc30c1fa700 -1 osd.2 94 *** Got signal Terminated *** 2024-06-11T14:01:51.748 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:51 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[42132]: 2024-06-11T14:01:51.492+0000 7fc30c1fa700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:01:52.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:52 smithi069 ceph-mon[66891]: osd.2 marked itself down and dead 2024-06-11T14:01:52.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:52 smithi135 ceph-mon[53782]: osd.2 marked itself down and dead 2024-06-11T14:01:53.093 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:52 smithi069 podman[77687]: 2024-06-11 14:01:52.78086491 +0000 UTC m=+1.388129571 container died fc965113df05c5cc6ef4a4dabb4ba9db4814b10ffe57a00c10519f76edb611d7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , release=754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, ceph=True, io.buildah.version=1.19.8, version=8, com.redhat.component=centos-stream-container, name=centos-stream) 2024-06-11T14:01:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:53 smithi069 ceph-mon[66891]: pgmap v82: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:53.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:53 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:01:53.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:53 smithi069 ceph-mon[66891]: osdmap e95: 8 total, 7 up, 8 in 2024-06-11T14:01:53.500 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:53 smithi069 podman[77687]: 2024-06-11 14:01:53.29350406 +0000 UTC m=+1.900768711 container cleanup fc965113df05c5cc6ef4a4dabb4ba9db4814b10ffe57a00c10519f76edb611d7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, ceph=True, distribution-scope=public, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vendor=Red Hat, Inc.) 2024-06-11T14:01:53.500 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:53 smithi069 bash[77687]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2 2024-06-11T14:01:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:53 smithi135 ceph-mon[53782]: pgmap v82: 97 pgs: 97 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:01:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:53 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:01:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:53 smithi135 ceph-mon[53782]: osdmap e95: 8 total, 7 up, 8 in 2024-06-11T14:01:53.857 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:53 smithi069 podman[77703]: 2024-06-11 14:01:53.536519231 +0000 UTC m=+0.755414608 container remove fc965113df05c5cc6ef4a4dabb4ba9db4814b10ffe57a00c10519f76edb611d7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, version=8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, distribution-scope=public, architecture=x86_64, vcs-type=git, ceph=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-06-11T14:01:54.155 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:53 smithi069 podman[77791]: 2024-06-11 14:01:53.761633674 +0000 UTC m=+0.017505300 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:01:54.156 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:53 smithi069 podman[77791]: 2024-06-11 14:01:53.862701842 +0000 UTC m=+0.118573492 container create 323af8bc1e778bc98da4c801f1bed5a410f90a985bc34f68cb802a4e2c6443f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-06-11T14:01:54.156 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 podman[77791]: 2024-06-11 14:01:54.081999919 +0000 UTC m=+0.337871529 container init 323af8bc1e778bc98da4c801f1bed5a410f90a985bc34f68cb802a4e2c6443f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-06-11T14:01:54.156 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 podman[77791]: 2024-06-11 14:01:54.085441089 +0000 UTC m=+0.341312688 container start 323af8bc1e778bc98da4c801f1bed5a410f90a985bc34f68cb802a4e2c6443f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:01:54.156 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 podman[77791]: 2024-06-11 14:01:54.156857863 +0000 UTC m=+0.412729462 container attach 323af8bc1e778bc98da4c801f1bed5a410f90a985bc34f68cb802a4e2c6443f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-06-11T14:01:54.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:54 smithi069 ceph-mon[66891]: osdmap e96: 8 total, 7 up, 8 in 2024-06-11T14:01:54.499 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 podman[77791]: 2024-06-11 14:01:54.35016833 +0000 UTC m=+0.606039939 container died 323af8bc1e778bc98da4c801f1bed5a410f90a985bc34f68cb802a4e2c6443f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-06-11T14:01:54.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:54 smithi135 ceph-mon[53782]: osdmap e96: 8 total, 7 up, 8 in 2024-06-11T14:01:54.957 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 podman[77814]: 2024-06-11 14:01:54.94374171 +0000 UTC m=+0.586148399 container remove 323af8bc1e778bc98da4c801f1bed5a410f90a985bc34f68cb802a4e2c6443f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:01:55.247 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:55 smithi069 ceph-mon[66891]: pgmap v85: 97 pgs: 5 stale+active+clean, 92 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:01:55.248 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.2.service: Deactivated successfully. 2024-06-11T14:01:55.248 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 systemd[1]: Stopped Ceph osd.2 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:01:55.248 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:54 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.2.service: Consumed 2min 41.029s CPU time. 2024-06-11T14:01:55.248 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:55 smithi069 systemd[1]: Starting Ceph osd.2 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:01:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:55 smithi135 ceph-mon[53782]: pgmap v85: 97 pgs: 5 stale+active+clean, 92 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:01:55.654 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:55 smithi069 podman[77913]: 2024-06-11 14:01:55.294113313 +0000 UTC m=+0.017787379 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:01:55.654 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:55 smithi069 podman[77913]: 2024-06-11 14:01:55.428073265 +0000 UTC m=+0.151747335 container create feb8fae03ce87d25c525651e108f756a906e668cd4b3516c35f3c59c4a60a9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate, GIT_CLEAN=True, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:01:55.654 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:55 smithi069 podman[77913]: 2024-06-11 14:01:55.655854643 +0000 UTC m=+0.379528699 container init feb8fae03ce87d25c525651e108f756a906e668cd4b3516c35f3c59c4a60a9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, GIT_CLEAN=True) 2024-06-11T14:01:55.998 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:55 smithi069 podman[77913]: 2024-06-11 14:01:55.659158596 +0000 UTC m=+0.382832650 container start feb8fae03ce87d25c525651e108f756a906e668cd4b3516c35f3c59c4a60a9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:01:55.998 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:55 smithi069 podman[77913]: 2024-06-11 14:01:55.739148194 +0000 UTC m=+0.462822254 container attach feb8fae03ce87d25c525651e108f756a906e668cd4b3516c35f3c59c4a60a9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3) 2024-06-11T14:01:56.434 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-mon[66891]: pgmap v86: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:01:56.434 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 41/426 objects degraded (9.624%), 10 pgs degraded (PG_DEGRADED) 2024-06-11T14:01:56.434 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-11T14:01:56.434 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-11T14:01:56.434 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-11T14:01:56.434 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-11T14:01:56.434 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate[77928]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 bash[77913]: --> ceph-volume raw activate successful for osd ID: 2 2024-06-11T14:01:56.435 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 podman[77913]: 2024-06-11 14:01:56.073354276 +0000 UTC m=+0.797028325 container died feb8fae03ce87d25c525651e108f756a906e668cd4b3516c35f3c59c4a60a9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528) 2024-06-11T14:01:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:56 smithi135 ceph-mon[53782]: pgmap v86: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:01:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:56 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 41/426 objects degraded (9.624%), 10 pgs degraded (PG_DEGRADED) 2024-06-11T14:01:57.056 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 podman[78051]: 2024-06-11 14:01:56.75945605 +0000 UTC m=+0.680642403 container remove feb8fae03ce87d25c525651e108f756a906e668cd4b3516c35f3c59c4a60a9df (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.35.3) 2024-06-11T14:01:57.057 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 podman[78109]: 2024-06-11 14:01:56.960381242 +0000 UTC m=+0.104410199 container create d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3) 2024-06-11T14:01:57.057 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:56 smithi069 podman[78109]: 2024-06-11 14:01:56.873529639 +0000 UTC m=+0.017558577 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:01:57.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:57 smithi069 podman[78109]: 2024-06-11 14:01:57.171328385 +0000 UTC m=+0.315357317 container init d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528) 2024-06-11T14:01:57.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:57 smithi069 podman[78109]: 2024-06-11 14:01:57.174667514 +0000 UTC m=+0.318696430 container start d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS) 2024-06-11T14:01:57.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:57 smithi069 ceph-osd[78129]: -- 172.21.15.69:0/1791303802 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cef3c081e0 con 0x55cef2e03c00 2024-06-11T14:01:57.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:57 smithi069 bash[78109]: d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 2024-06-11T14:01:57.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:57 smithi069 systemd[1]: Started Ceph osd.2 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:01:57.998 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:01:57 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:01:57.722+0000 7f3533b1a640 -1 Falling back to public interface 2024-06-11T14:01:58.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:58.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:58.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:01:58.658 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:01:58 smithi069 ceph-mon[66891]: pgmap v87: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:01:58.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:58.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:01:58.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:01:58.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:01:58 smithi135 ceph-mon[53782]: pgmap v87: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:01.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:01 smithi069 ceph-mon[66891]: pgmap v88: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:01.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:01 smithi135 ceph-mon[53782]: pgmap v88: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:02.709 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:02:02 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:02:02.608+0000 7f3533b1a640 -1 osd.2 0 read_superblock omap replica is missing. 2024-06-11T14:02:03.497 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:02:03 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:02:03.290+0000 7f3533b1a640 -1 osd.2 94 log_to_monitors true 2024-06-11T14:02:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:03 smithi069 ceph-mon[66891]: pgmap v89: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:03 smithi135 ceph-mon[53782]: pgmap v89: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:04 smithi069 ceph-mon[66891]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-11T14:02:04.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:04 smithi135 ceph-mon[53782]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-06-11T14:02:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:04.952 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:02:04 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:02:04.600+0000 7f35278bd640 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:02:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:05 smithi069 ceph-mon[66891]: pgmap v90: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:05 smithi069 ceph-mon[66891]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-11T14:02:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:05 smithi069 ceph-mon[66891]: osdmap e97: 8 total, 7 up, 8 in 2024-06-11T14:02:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:05 smithi069 ceph-mon[66891]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:02:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:05 smithi069 ceph-mon[66891]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' 2024-06-11T14:02:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:05 smithi135 ceph-mon[53782]: pgmap v90: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:05 smithi135 ceph-mon[53782]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-06-11T14:02:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:05 smithi135 ceph-mon[53782]: osdmap e97: 8 total, 7 up, 8 in 2024-06-11T14:02:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:05 smithi135 ceph-mon[53782]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:02:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:05 smithi135 ceph-mon[53782]: from='osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090]' entity='osd.2' 2024-06-11T14:02:06.460 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:06 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:02:06.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:06 smithi069 ceph-mon[66891]: osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090] boot 2024-06-11T14:02:06.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:06 smithi069 ceph-mon[66891]: osdmap e98: 8 total, 8 up, 8 in 2024-06-11T14:02:06.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T14:02:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:06 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:02:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:06 smithi135 ceph-mon[53782]: osd.2 [v2:172.21.15.69:6812/2512901090,v1:172.21.15.69:6813/2512901090] boot 2024-06-11T14:02:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:06 smithi135 ceph-mon[53782]: osdmap e98: 8 total, 8 up, 8 in 2024-06-11T14:02:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-06-11T14:02:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: pgmap v93: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/426 objects degraded (9.624%) 2024-06-11T14:02:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: osdmap e99: 8 total, 8 up, 8 in 2024-06-11T14:02:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:07.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: pgmap v93: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 41/426 objects degraded (9.624%) 2024-06-11T14:02:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: osdmap e99: 8 total, 8 up, 8 in 2024-06-11T14:02:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:07.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:08 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:08 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-11T14:02:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:08 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:08 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-06-11T14:02:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:09 smithi069 ceph-mon[66891]: pgmap v95: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:09 smithi135 ceph-mon[53782]: pgmap v95: 97 pgs: 15 active+undersized, 10 active+undersized+degraded, 72 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:02:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:10 smithi069 ceph-mon[66891]: pgmap v96: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:02:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:10 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/426 objects degraded (9.624%), 10 pgs degraded) 2024-06-11T14:02:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:10 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:02:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:10 smithi135 ceph-mon[53782]: pgmap v96: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:02:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:10 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/426 objects degraded (9.624%), 10 pgs degraded) 2024-06-11T14:02:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:10 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:02:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:13 smithi069 ceph-mon[66891]: pgmap v97: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 257 B/s rd, 0 op/s 2024-06-11T14:02:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:13 smithi135 ceph-mon[53782]: pgmap v97: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 257 B/s rd, 0 op/s 2024-06-11T14:02:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:15 smithi069 ceph-mon[66891]: pgmap v98: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 240 B/s rd, 0 op/s 2024-06-11T14:02:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:15 smithi135 ceph-mon[53782]: pgmap v98: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 240 B/s rd, 0 op/s 2024-06-11T14:02:17.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:17 smithi069 ceph-mon[66891]: pgmap v99: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:02:17.471 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:02:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:17 smithi135 ceph-mon[53782]: pgmap v99: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:02:18.152 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:02:18.152 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (3m) 15s ago 25m 17.4M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (115s) 15s ago 25m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (111s) 107s ago 25m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (3m) 15s ago 25m 43.2M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (20m) 15s ago 20m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (20m) 107s ago 20m 19.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (3m) 15s ago 27m 501M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (3m) 107s ago 25m 438M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (2m) 15s ago 27m 47.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (2m) 107s ago 25m 34.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (19m) 15s ago 19m 296M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (19m) 107s ago 19m 87.4M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (3m) 15s ago 25m 8833k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:02:18.153 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (3m) 107s ago 24m 16.3M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (91s) 15s ago 24m 207M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (56s) 15s ago 24m 180M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (21s) 15s ago 23m 14.4M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (23m) 15s ago 23m 413M 1936M 17.2.0 e1d6a67b021e 99de7727854a 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (22m) 107s ago 22m 459M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (22m) 107s ago 22m 534M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (21m) 107s ago 21m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (21m) 107s ago 21m 689M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:02:18.154 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (3m) 15s ago 24m 74.0M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:02:18.557 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 3 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:18.558 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 7 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:02:18.559 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:02:18.890 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:02:18.890 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:02:18.890 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:02:18.890 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "9/23 daemons upgraded", 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:02:18.891 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:02:19.288 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: pgmap v100: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 178 B/s rd, 0 op/s 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3611149478' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: pgmap v100: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 178 B/s rd, 0 op/s 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3611149478' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:20 smithi069 ceph-mon[66891]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:20 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2697216608' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:02:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:20 smithi135 ceph-mon[53782]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:20 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2697216608' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:02:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:21 smithi069 ceph-mon[66891]: pgmap v101: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:02:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:21 smithi135 ceph-mon[53782]: pgmap v101: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:02:22.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:22 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:22.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:22 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:23.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:23 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:23.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:23 smithi069 ceph-mon[66891]: Upgrade: osd.3 is safe to restart 2024-06-11T14:02:23.224 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:23 smithi069 ceph-mon[66891]: pgmap v102: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:23 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-06-11T14:02:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:23 smithi135 ceph-mon[53782]: Upgrade: osd.3 is safe to restart 2024-06-11T14:02:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:23 smithi135 ceph-mon[53782]: pgmap v102: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:24 smithi135 ceph-mon[53782]: Upgrade: Updating osd.3 2024-06-11T14:02:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:24 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:24 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-11T14:02:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:24 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:24 smithi135 ceph-mon[53782]: Deploying daemon osd.3 on smithi069 2024-06-11T14:02:24.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:24 smithi135 ceph-mon[53782]: pgmap v103: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:24.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:24 smithi069 ceph-mon[66891]: Upgrade: Updating osd.3 2024-06-11T14:02:24.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:24 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:24.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:24 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-06-11T14:02:24.636 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:24 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:24.637 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:24 smithi069 ceph-mon[66891]: Deploying daemon osd.3 on smithi069 2024-06-11T14:02:24.637 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:24 smithi069 ceph-mon[66891]: pgmap v103: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:26.485 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:26 smithi069 systemd[1]: Stopping Ceph osd.3 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:02:26.748 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:26 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T14:02:26.485+0000 7f596596c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:02:26.748 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:26 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T14:02:26.485+0000 7f596596c700 -1 osd.3 99 *** Got signal Terminated *** 2024-06-11T14:02:26.748 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:26 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[45881]: 2024-06-11T14:02:26.485+0000 7f596596c700 -1 osd.3 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:02:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:27 smithi069 ceph-mon[66891]: pgmap v104: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:27 smithi069 ceph-mon[66891]: osd.3 marked itself down and dead 2024-06-11T14:02:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:27 smithi135 ceph-mon[53782]: pgmap v104: 97 pgs: 97 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:27 smithi135 ceph-mon[53782]: osd.3 marked itself down and dead 2024-06-11T14:02:28.043 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:27 smithi069 podman[80308]: 2024-06-11 14:02:27.783489472 +0000 UTC m=+1.416349630 container died 99de7727854a9fcd78651e1c3d11b50d7e30e3e1c4a24457393edc7d28209d50 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, version=8, name=centos-stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD) 2024-06-11T14:02:28.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:28 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:02:28.314 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:28 smithi069 ceph-mon[66891]: osdmap e100: 8 total, 7 up, 8 in 2024-06-11T14:02:28.314 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:28 smithi069 podman[80308]: 2024-06-11 14:02:28.202101671 +0000 UTC m=+1.834961852 container cleanup 99de7727854a9fcd78651e1c3d11b50d7e30e3e1c4a24457393edc7d28209d50 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, io.openshift.tags=base centos centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.component=centos-stream-container, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public) 2024-06-11T14:02:28.314 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:28 smithi069 bash[80308]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3 2024-06-11T14:02:28.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:28 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:02:28.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:28 smithi135 ceph-mon[53782]: osdmap e100: 8 total, 7 up, 8 in 2024-06-11T14:02:28.658 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:28 smithi069 podman[80323]: 2024-06-11 14:02:28.395299049 +0000 UTC m=+0.612074025 container remove 99de7727854a9fcd78651e1c3d11b50d7e30e3e1c4a24457393edc7d28209d50 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, distribution-scope=public, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=HEAD, io.buildah.version=1.19.8, version=8, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-06-11T14:02:28.928 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:28 smithi069 podman[80411]: 2024-06-11 14:02:28.721601486 +0000 UTC m=+0.017230657 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:02:28.929 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:28 smithi069 podman[80411]: 2024-06-11 14:02:28.821359611 +0000 UTC m=+0.116988762 container create f8c90b9a643283fb3af62c5f1b164e0f1c683ff897021bcccb291b493bc899ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.35.3, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:02:29.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:29 smithi069 ceph-mon[66891]: pgmap v106: 97 pgs: 16 stale+active+clean, 81 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:02:29.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:29 smithi069 ceph-mon[66891]: osdmap e101: 8 total, 7 up, 8 in 2024-06-11T14:02:29.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 podman[80411]: 2024-06-11 14:02:29.032441199 +0000 UTC m=+0.328070367 container init f8c90b9a643283fb3af62c5f1b164e0f1c683ff897021bcccb291b493bc899ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-06-11T14:02:29.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 podman[80411]: 2024-06-11 14:02:29.036136013 +0000 UTC m=+0.331765166 container start f8c90b9a643283fb3af62c5f1b164e0f1c683ff897021bcccb291b493bc899ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, ceph=True) 2024-06-11T14:02:29.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 podman[80411]: 2024-06-11 14:02:29.115441806 +0000 UTC m=+0.411070970 container attach f8c90b9a643283fb3af62c5f1b164e0f1c683ff897021bcccb291b493bc899ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3) 2024-06-11T14:02:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:29 smithi135 ceph-mon[53782]: pgmap v106: 97 pgs: 16 stale+active+clean, 81 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:02:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:29 smithi135 ceph-mon[53782]: osdmap e101: 8 total, 7 up, 8 in 2024-06-11T14:02:29.627 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 podman[80411]: 2024-06-11 14:02:29.290062441 +0000 UTC m=+0.585691593 container died f8c90b9a643283fb3af62c5f1b164e0f1c683ff897021bcccb291b493bc899ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3) 2024-06-11T14:02:30.218 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 podman[80433]: 2024-06-11 14:02:29.902385164 +0000 UTC m=+0.606875346 container remove f8c90b9a643283fb3af62c5f1b164e0f1c683ff897021bcccb291b493bc899ae (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-06-11T14:02:30.219 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.3.service: Deactivated successfully. 2024-06-11T14:02:30.219 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 systemd[1]: Stopped Ceph osd.3 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:02:30.219 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:29 smithi069 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.3.service: Consumed 4min 18.400s CPU time. 2024-06-11T14:02:30.219 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 systemd[1]: Starting Ceph osd.3 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:02:30.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-mon[66891]: pgmap v108: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:30.485 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 51/426 objects degraded (11.972%), 14 pgs degraded (PG_DEGRADED) 2024-06-11T14:02:30.485 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 podman[80530]: 2024-06-11 14:02:30.32831526 +0000 UTC m=+0.106755172 container create dcd95baa970cf9ea53273087efcd00d7f751b2fbd28f589c8948113abe03fe05 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, ceph=True) 2024-06-11T14:02:30.486 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 podman[80530]: 2024-06-11 14:02:30.239068419 +0000 UTC m=+0.017508343 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:02:30.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:30 smithi135 ceph-mon[53782]: pgmap v108: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:30.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:30 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 51/426 objects degraded (11.972%), 14 pgs degraded (PG_DEGRADED) 2024-06-11T14:02:30.964 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 podman[80530]: 2024-06-11 14:02:30.597731499 +0000 UTC m=+0.376171419 container init dcd95baa970cf9ea53273087efcd00d7f751b2fbd28f589c8948113abe03fe05 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3) 2024-06-11T14:02:30.964 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 podman[80530]: 2024-06-11 14:02:30.601565877 +0000 UTC m=+0.380005793 container start dcd95baa970cf9ea53273087efcd00d7f751b2fbd28f589c8948113abe03fe05 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, ceph=True) 2024-06-11T14:02:30.964 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 podman[80530]: 2024-06-11 14:02:30.764706999 +0000 UTC m=+0.543146925 container attach dcd95baa970cf9ea53273087efcd00d7f751b2fbd28f589c8948113abe03fe05 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, GIT_CLEAN=True, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-06-11T14:02:31.248 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-11T14:02:31.248 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-11T14:02:31.248 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-11T14:02:31.248 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-11T14:02:31.248 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-11T14:02:31.248 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate[80546]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:30 smithi069 bash[80530]: --> ceph-volume raw activate successful for osd ID: 3 2024-06-11T14:02:31.249 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:31 smithi069 podman[80530]: 2024-06-11 14:02:31.020092978 +0000 UTC m=+0.798532892 container died dcd95baa970cf9ea53273087efcd00d7f751b2fbd28f589c8948113abe03fe05 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-06-11T14:02:31.882 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:31 smithi069 podman[80680]: 2024-06-11 14:02:31.601653504 +0000 UTC m=+0.575793502 container remove dcd95baa970cf9ea53273087efcd00d7f751b2fbd28f589c8948113abe03fe05 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.license=GPLv2) 2024-06-11T14:02:31.883 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:31 smithi069 podman[80741]: 2024-06-11 14:02:31.794194057 +0000 UTC m=+0.097065947 container create e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:02:31.883 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:31 smithi069 podman[80741]: 2024-06-11 14:02:31.714334858 +0000 UTC m=+0.017206755 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:02:32.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:32 smithi069 podman[80741]: 2024-06-11 14:02:32.004930265 +0000 UTC m=+0.307802164 container init e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.35.3) 2024-06-11T14:02:32.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:32 smithi069 podman[80741]: 2024-06-11 14:02:32.008133628 +0000 UTC m=+0.311005518 container start e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:02:32.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:32 smithi069 ceph-osd[80759]: -- 172.21.15.69:0/2165278813 <== mon.0 v2:172.21.15.69:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564f591e61e0 con 0x564f591c2000 2024-06-11T14:02:32.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:32 smithi069 bash[80741]: e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd 2024-06-11T14:02:32.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:32 smithi069 systemd[1]: Started Ceph osd.3 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:02:32.844 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:32 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:02:32.548+0000 7fb6b6b2a640 -1 Falling back to public interface 2024-06-11T14:02:33.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:33 smithi069 ceph-mon[66891]: pgmap v109: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:33.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:33.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:33.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:02:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:33 smithi135 ceph-mon[53782]: pgmap v109: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:02:35.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:35 smithi069 ceph-mon[66891]: pgmap v110: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:35.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:35.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:35 smithi135 ceph-mon[53782]: pgmap v110: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:37.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:37 smithi069 ceph-mon[66891]: pgmap v111: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:37 smithi135 ceph-mon[53782]: pgmap v111: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:38.247 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:37 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:02:37.840+0000 7fb6b6b2a640 -1 osd.3 0 read_superblock omap replica is missing. 2024-06-11T14:02:38.574 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:38.574 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:38.574 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:38 smithi069 ceph-mon[66891]: pgmap v112: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:38.574 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:38.574 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:38.830 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:38 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:02:38.811+0000 7fb6b6b2a640 -1 osd.3 99 log_to_monitors true 2024-06-11T14:02:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:38 smithi135 ceph-mon[53782]: pgmap v112: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 51/426 objects degraded (11.972%) 2024-06-11T14:02:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:39.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:39.840 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:39 smithi069 ceph-mon[66891]: from='osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-11T14:02:39.841 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:02:39 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:02:39.610+0000 7fb6ae8d5640 -1 osd.3 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:02:40.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:39 smithi135 ceph-mon[53782]: from='osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-06-11T14:02:40.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:40 smithi069 ceph-mon[66891]: from='osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-11T14:02:40.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:40 smithi069 ceph-mon[66891]: osdmap e102: 8 total, 7 up, 8 in 2024-06-11T14:02:40.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:40 smithi069 ceph-mon[66891]: from='osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:02:40.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:40 smithi069 ceph-mon[66891]: pgmap v114: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 51/426 objects degraded (11.972%) 2024-06-11T14:02:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:40 smithi135 ceph-mon[53782]: from='osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-06-11T14:02:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:40 smithi135 ceph-mon[53782]: osdmap e102: 8 total, 7 up, 8 in 2024-06-11T14:02:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:40 smithi135 ceph-mon[53782]: from='osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi069", "root=default"]}]: dispatch 2024-06-11T14:02:41.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:40 smithi135 ceph-mon[53782]: pgmap v114: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 51/426 objects degraded (11.972%) 2024-06-11T14:02:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:02:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643] boot 2024-06-11T14:02:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: osdmap e103: 8 total, 8 up, 8 in 2024-06-11T14:02:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T14:02:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:41.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:42.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:02:42.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: osd.3 [v2:172.21.15.69:6826/3447381643,v1:172.21.15.69:6827/3447381643] boot 2024-06-11T14:02:42.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: osdmap e103: 8 total, 8 up, 8 in 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:42.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:42.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:42.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:42 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:42.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:42 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-11T14:02:42.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:42 smithi069 ceph-mon[66891]: osdmap e104: 8 total, 8 up, 8 in 2024-06-11T14:02:42.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:42 smithi069 ceph-mon[66891]: pgmap v117: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 51/426 objects degraded (11.972%) 2024-06-11T14:02:43.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:42 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:43.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:42 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-11T14:02:43.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:42 smithi135 ceph-mon[53782]: osdmap e104: 8 total, 8 up, 8 in 2024-06-11T14:02:43.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:42 smithi135 ceph-mon[53782]: pgmap v117: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 51/426 objects degraded (11.972%) 2024-06-11T14:02:45.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:45 smithi069 ceph-mon[66891]: pgmap v118: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 51/426 objects degraded (11.972%) 2024-06-11T14:02:45.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:45 smithi135 ceph-mon[53782]: pgmap v118: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 51/426 objects degraded (11.972%) 2024-06-11T14:02:46.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:46 smithi069 ceph-mon[66891]: pgmap v119: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 314 B/s rd, 0 op/s 2024-06-11T14:02:46.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:46 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/426 objects degraded (11.972%), 14 pgs degraded) 2024-06-11T14:02:46.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:46 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:02:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:46 smithi135 ceph-mon[53782]: pgmap v119: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 314 B/s rd, 0 op/s 2024-06-11T14:02:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:46 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/426 objects degraded (11.972%), 14 pgs degraded) 2024-06-11T14:02:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:46 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:02:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:49 smithi069 ceph-mon[66891]: pgmap v120: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-06-11T14:02:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:49 smithi135 ceph-mon[53782]: pgmap v120: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-06-11T14:02:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:02:49.652 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:02:50.331 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:02:50.331 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (4m) 12s ago 26m 16.1M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:02:50.331 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (2m) 12s ago 26m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:02:50.331 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (2m) 2m ago 25m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:02:50.331 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (4m) 12s ago 26m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (20m) 12s ago 20m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (20m) 2m ago 20m 19.0M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (4m) 12s ago 27m 502M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (3m) 2m ago 25m 438M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (3m) 12s ago 27m 47.8M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (3m) 2m ago 25m 34.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (20m) 12s ago 20m 298M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (20m) 2m ago 20m 87.4M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (4m) 12s ago 26m 8996k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (4m) 2m ago 25m 16.3M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (2m) 12s ago 25m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (88s) 12s ago 24m 181M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:02:50.332 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (53s) 12s ago 24m 162M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:02:50.333 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (18s) 12s ago 23m 14.4M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:02:50.333 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (23m) 2m ago 23m 459M 2704M 17.2.0 e1d6a67b021e 858b8350f61a 2024-06-11T14:02:50.333 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (22m) 2m ago 22m 534M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:02:50.333 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (22m) 2m ago 22m 456M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:02:50.333 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (22m) 2m ago 22m 689M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:02:50.333 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (3m) 12s ago 25m 76.4M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:02:50.734 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:02:50.734 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:02:50.734 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:02:50.734 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:50.734 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 4 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-06-11T14:02:50.735 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:02:50.736 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:02:50.736 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:02:51.065 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "10/23 daemons upgraded", 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:02:51.066 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:02:51.067 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:02:51.461 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:02:51.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:51 smithi069 ceph-mon[66891]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:51.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:51 smithi069 ceph-mon[66891]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:51.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:51 smithi069 ceph-mon[66891]: pgmap v121: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 0 op/s 2024-06-11T14:02:51.462 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:51 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1787679684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:51 smithi135 ceph-mon[53782]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:51 smithi135 ceph-mon[53782]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:51 smithi135 ceph-mon[53782]: pgmap v121: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 215 B/s rd, 0 op/s 2024-06-11T14:02:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:51 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1787679684' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:02:52.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:52 smithi069 ceph-mon[66891]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:52.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:52 smithi069 ceph-mon[66891]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:52.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:52 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2574317908' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:02:52.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:52 smithi135 ceph-mon[53782]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:52.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:52 smithi135 ceph-mon[53782]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:02:52.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:52 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2574317908' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:02:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:53 smithi069 ceph-mon[66891]: pgmap v122: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-06-11T14:02:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:53 smithi135 ceph-mon[53782]: pgmap v122: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 194 B/s rd, 0 op/s 2024-06-11T14:02:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:55 smithi069 ceph-mon[66891]: pgmap v123: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:55 smithi135 ceph-mon[53782]: pgmap v123: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:57.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:57 smithi135 ceph-mon[53782]: pgmap v124: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:57.296 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:57 smithi069 ceph-mon[66891]: pgmap v124: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:58 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:58 smithi135 ceph-mon[53782]: Upgrade: osd.4 is safe to restart 2024-06-11T14:02:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-11T14:02:58.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:58 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:58 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-06-11T14:02:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:58 smithi069 ceph-mon[66891]: Upgrade: osd.4 is safe to restart 2024-06-11T14:02:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:02:58.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-06-11T14:02:58.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:58 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:02:59.476 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:59 smithi135 ceph-mon[53782]: Upgrade: Updating osd.4 2024-06-11T14:02:59.476 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:59 smithi135 ceph-mon[53782]: Deploying daemon osd.4 on smithi135 2024-06-11T14:02:59.477 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:02:59 smithi135 ceph-mon[53782]: pgmap v125: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:02:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:59 smithi069 ceph-mon[66891]: Upgrade: Updating osd.4 2024-06-11T14:02:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:59 smithi069 ceph-mon[66891]: Deploying daemon osd.4 on smithi135 2024-06-11T14:02:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:02:59 smithi069 ceph-mon[66891]: pgmap v125: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:01.448 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:01 smithi135 ceph-mon[53782]: pgmap v126: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:03:01.448 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:01 smithi135 systemd[1]: Stopping Ceph osd.4 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:03:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:01 smithi069 ceph-mon[66891]: pgmap v126: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:03:01.797 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T14:03:01.447+0000 7f9442737700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:03:01.798 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T14:03:01.447+0000 7f9442737700 -1 osd.4 104 *** Got signal Terminated *** 2024-06-11T14:03:01.798 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[31711]: 2024-06-11T14:03:01.447+0000 7f9442737700 -1 osd.4 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:03:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:02 smithi069 ceph-mon[66891]: osd.4 marked itself down and dead 2024-06-11T14:03:02.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:02 smithi135 ceph-mon[53782]: osd.4 marked itself down and dead 2024-06-11T14:03:03.023 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:02 smithi135 podman[58133]: 2024-06-11 14:03:02.755305387 +0000 UTC m=+1.428277818 container died 858b8350f61ae0676bd84f8d756865910230596f0e7f3e3f9b96b61661cdce15 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, version=8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, ceph=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-06-11T14:03:03.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:03 smithi135 ceph-mon[53782]: pgmap v127: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:03.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:03 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:03:03.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:03 smithi135 ceph-mon[53782]: osdmap e105: 8 total, 7 up, 8 in 2024-06-11T14:03:03.298 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 podman[58133]: 2024-06-11 14:03:03.131026731 +0000 UTC m=+1.803999163 container cleanup 858b8350f61ae0676bd84f8d756865910230596f0e7f3e3f9b96b61661cdce15 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, release=754, RELEASE=HEAD, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream) 2024-06-11T14:03:03.298 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 bash[58133]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4 2024-06-11T14:03:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:03 smithi069 ceph-mon[66891]: pgmap v127: 97 pgs: 97 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:03 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:03:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:03 smithi069 ceph-mon[66891]: osdmap e105: 8 total, 7 up, 8 in 2024-06-11T14:03:03.721 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 podman[58152]: 2024-06-11 14:03:03.395802756 +0000 UTC m=+0.640977957 container remove 858b8350f61ae0676bd84f8d756865910230596f0e7f3e3f9b96b61661cdce15 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, release=754, io.buildah.version=1.19.8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-06-11T14:03:03.993 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 podman[58272]: 2024-06-11 14:03:03.625244461 +0000 UTC m=+0.017224255 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:03:03.993 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 podman[58272]: 2024-06-11 14:03:03.733663289 +0000 UTC m=+0.125643078 container create 957900df079bd2cac547b8c6f05eff1c38164ad138d44f2f77e724bcb0905e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:03:03.993 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 podman[58272]: 2024-06-11 14:03:03.994344057 +0000 UTC m=+0.386323827 container init 957900df079bd2cac547b8c6f05eff1c38164ad138d44f2f77e724bcb0905e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-06-11T14:03:04.254 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:04 smithi135 ceph-mon[53782]: osdmap e106: 8 total, 7 up, 8 in 2024-06-11T14:03:04.255 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:04.255 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:04.255 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:03 smithi135 podman[58272]: 2024-06-11 14:03:03.997670139 +0000 UTC m=+0.389649908 container start 957900df079bd2cac547b8c6f05eff1c38164ad138d44f2f77e724bcb0905e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, ceph=True) 2024-06-11T14:03:04.255 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:04 smithi135 podman[58272]: 2024-06-11 14:03:04.077598822 +0000 UTC m=+0.469578597 container attach 957900df079bd2cac547b8c6f05eff1c38164ad138d44f2f77e724bcb0905e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3) 2024-06-11T14:03:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:04 smithi069 ceph-mon[66891]: osdmap e106: 8 total, 7 up, 8 in 2024-06-11T14:03:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:04.547 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:04 smithi135 podman[58272]: 2024-06-11 14:03:04.256348421 +0000 UTC m=+0.648328193 container died 957900df079bd2cac547b8c6f05eff1c38164ad138d44f2f77e724bcb0905e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-06-11T14:03:05.204 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:04 smithi135 podman[58295]: 2024-06-11 14:03:04.90608859 +0000 UTC m=+0.640110515 container remove 957900df079bd2cac547b8c6f05eff1c38164ad138d44f2f77e724bcb0905e59 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-06-11T14:03:05.205 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:04 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.4.service: Deactivated successfully. 2024-06-11T14:03:05.205 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:04 smithi135 systemd[1]: Stopped Ceph osd.4 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:03:05.205 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:04 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.4.service: Consumed 4min 12.154s CPU time. 2024-06-11T14:03:05.205 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 systemd[1]: Starting Ceph osd.4 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:03:05.461 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-mon[53782]: pgmap v130: 97 pgs: 15 stale+active+clean, 82 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:03:05.462 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 podman[58394]: 2024-06-11 14:03:05.243173995 +0000 UTC m=+0.017360026 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:03:05.462 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 podman[58394]: 2024-06-11 14:03:05.35773605 +0000 UTC m=+0.131922070 container create c3c6460bf22bc73ed34aaf41f3168cc1f33bc05447f93a4377ecb1f47ab579eb (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.schema-version=1.0) 2024-06-11T14:03:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:05 smithi069 ceph-mon[66891]: pgmap v130: 97 pgs: 15 stale+active+clean, 82 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:03:05.797 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 podman[58394]: 2024-06-11 14:03:05.568490753 +0000 UTC m=+0.342676779 container init c3c6460bf22bc73ed34aaf41f3168cc1f33bc05447f93a4377ecb1f47ab579eb (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS) 2024-06-11T14:03:05.797 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 podman[58394]: 2024-06-11 14:03:05.571849833 +0000 UTC m=+0.346035863 container start c3c6460bf22bc73ed34aaf41f3168cc1f33bc05447f93a4377ecb1f47ab579eb (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528) 2024-06-11T14:03:05.797 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 podman[58394]: 2024-06-11 14:03:05.643456587 +0000 UTC m=+0.417642611 container attach c3c6460bf22bc73ed34aaf41f3168cc1f33bc05447f93a4377ecb1f47ab579eb (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-06-11T14:03:06.209 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-11T14:03:06.209 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-11T14:03:06.209 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-11T14:03:06.209 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-06-11T14:03:06.209 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-11T14:03:06.210 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-06-11T14:03:06.210 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-11T14:03:06.210 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-06-11T14:03:06.210 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-11T14:03:06.210 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-06-11T14:03:06.210 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-11T14:03:06.211 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-06-11T14:03:06.211 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate[58409]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-11T14:03:06.211 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 bash[58394]: --> ceph-volume raw activate successful for osd ID: 4 2024-06-11T14:03:06.211 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:05 smithi135 podman[58394]: 2024-06-11 14:03:05.990993758 +0000 UTC m=+0.765179788 container died c3c6460bf22bc73ed34aaf41f3168cc1f33bc05447f93a4377ecb1f47ab579eb (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux ) 2024-06-11T14:03:06.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:06 smithi069 ceph-mon[66891]: pgmap v131: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:06 smithi135 ceph-mon[53782]: pgmap v131: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:07.007 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:06 smithi135 podman[58521]: 2024-06-11 14:03:06.664258442 +0000 UTC m=+0.667627063 container remove c3c6460bf22bc73ed34aaf41f3168cc1f33bc05447f93a4377ecb1f47ab579eb (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=) 2024-06-11T14:03:07.007 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:06 smithi135 podman[58578]: 2024-06-11 14:03:06.782339486 +0000 UTC m=+0.016954558 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:03:07.007 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:06 smithi135 podman[58578]: 2024-06-11 14:03:06.898742291 +0000 UTC m=+0.133357392 container create 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:03:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:07 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 45/426 objects degraded (10.563%), 12 pgs degraded (PG_DEGRADED) 2024-06-11T14:03:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:07 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 45/426 objects degraded (10.563%), 12 pgs degraded (PG_DEGRADED) 2024-06-11T14:03:07.547 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:07 smithi135 podman[58578]: 2024-06-11 14:03:07.125730521 +0000 UTC m=+0.360345580 container init 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, io.buildah.version=1.35.3, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:03:07.547 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:07 smithi135 podman[58578]: 2024-06-11 14:03:07.128755511 +0000 UTC m=+0.363370567 container start 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:03:07.547 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:07 smithi135 ceph-osd[58597]: -- 172.21.15.135:0/3798748101 <== mon.1 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e87dd401e0 con 0x55e87cf3bc00 2024-06-11T14:03:07.547 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:07 smithi135 bash[58578]: 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 2024-06-11T14:03:07.548 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:07 smithi135 systemd[1]: Started Ceph osd.4 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:03:07.919 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:03:07.664+0000 7f4b09e4d640 -1 Falling back to public interface 2024-06-11T14:03:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:08 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:08 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:08 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:08 smithi069 ceph-mon[66891]: pgmap v132: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45/426 objects degraded (10.563%) 2024-06-11T14:03:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:08 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:08 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:08 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:08 smithi135 ceph-mon[53782]: pgmap v132: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45/426 objects degraded (10.563%) 2024-06-11T14:03:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:11 smithi069 ceph-mon[66891]: pgmap v133: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:11 smithi135 ceph-mon[53782]: pgmap v133: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:12.846 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:03:12.792+0000 7f4b09e4d640 -1 osd.4 0 read_superblock omap replica is missing. 2024-06-11T14:03:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:13 smithi069 ceph-mon[66891]: pgmap v134: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 206 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:13 smithi135 ceph-mon[53782]: pgmap v134: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 206 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:14.174 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:03:13.995+0000 7f4b09e4d640 -1 osd.4 104 log_to_monitors true 2024-06-11T14:03:14.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:14 smithi069 ceph-mon[66891]: from='osd.4 [v2:172.21.15.135:6800/565700960,v1:172.21.15.135:6801/565700960]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T14:03:14.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:14 smithi069 ceph-mon[66891]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T14:03:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:14 smithi135 ceph-mon[53782]: from='osd.4 [v2:172.21.15.135:6800/565700960,v1:172.21.15.135:6801/565700960]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T14:03:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:14 smithi135 ceph-mon[53782]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-06-11T14:03:15.047 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:03:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:03:14.637+0000 7f4afdbf0640 -1 osd.4 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:03:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: pgmap v135: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 187 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-11T14:03:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: osdmap e107: 8 total, 7 up, 8 in 2024-06-11T14:03:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='osd.4 [v2:172.21.15.135:6800/565700960,v1:172.21.15.135:6801/565700960]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='osd.4 ' entity='osd.4' 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: pgmap v135: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 187 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-06-11T14:03:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: osdmap e107: 8 total, 7 up, 8 in 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='osd.4 [v2:172.21.15.135:6800/565700960,v1:172.21.15.135:6801/565700960]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='osd.4 ' entity='osd.4' 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:15.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:16.458 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:16 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:16.459 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:16 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-06-11T14:03:16.459 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:16 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:03:16.459 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:16 smithi135 ceph-mon[53782]: osd.4 [v2:172.21.15.135:6800/565700960,v1:172.21.15.135:6801/565700960] boot 2024-06-11T14:03:16.459 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:16 smithi135 ceph-mon[53782]: osdmap e108: 8 total, 8 up, 8 in 2024-06-11T14:03:16.459 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T14:03:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:16 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:16 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-06-11T14:03:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:16 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:03:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:16 smithi069 ceph-mon[66891]: osd.4 [v2:172.21.15.135:6800/565700960,v1:172.21.15.135:6801/565700960] boot 2024-06-11T14:03:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:16 smithi069 ceph-mon[66891]: osdmap e108: 8 total, 8 up, 8 in 2024-06-11T14:03:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-06-11T14:03:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:17 smithi069 ceph-mon[66891]: pgmap v138: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:17 smithi069 ceph-mon[66891]: osdmap e109: 8 total, 8 up, 8 in 2024-06-11T14:03:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:17 smithi135 ceph-mon[53782]: pgmap v138: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/426 objects degraded (10.563%) 2024-06-11T14:03:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:17 smithi135 ceph-mon[53782]: osdmap e109: 8 total, 8 up, 8 in 2024-06-11T14:03:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:19 smithi069 ceph-mon[66891]: pgmap v140: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 45/426 objects degraded (10.563%) 2024-06-11T14:03:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:19 smithi135 ceph-mon[53782]: pgmap v140: 97 pgs: 30 active+undersized, 12 active+undersized+degraded, 55 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 45/426 objects degraded (10.563%) 2024-06-11T14:03:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:20 smithi069 ceph-mon[66891]: pgmap v141: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:03:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:20 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/426 objects degraded (10.563%), 12 pgs degraded) 2024-06-11T14:03:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:20 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:03:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:20 smithi135 ceph-mon[53782]: pgmap v141: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:03:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:20 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/426 objects degraded (10.563%), 12 pgs degraded) 2024-06-11T14:03:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:20 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:03:21.826 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:03:22.472 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:03:22.472 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (4m) 44s ago 27m 16.1M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (3m) 44s ago 26m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (2m) 11s ago 26m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (4m) 44s ago 26m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (21m) 44s ago 21m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (21m) 11s ago 21m 19.5M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (4m) 44s ago 28m 502M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (4m) 11s ago 26m 439M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (3m) 44s ago 28m 47.8M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (3m) 11s ago 26m 39.3M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (20m) 44s ago 20m 298M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (20m) 11s ago 20m 87.6M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (4m) 44s ago 26m 8996k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:03:22.473 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (4m) 11s ago 26m 15.6M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (2m) 44s ago 25m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (2m) 44s ago 25m 181M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (85s) 44s ago 24m 162M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (50s) 44s ago 24m 14.4M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (15s) 11s ago 23m 12.0M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (23m) 11s ago 23m 531M 2704M 17.2.0 e1d6a67b021e 0528d0439ea6 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (23m) 11s ago 23m 454M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (22m) 11s ago 22m 688M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:03:22.474 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (4m) 44s ago 25m 76.4M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:03:22.866 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:03:22.866 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 5 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-06-11T14:03:22.867 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 9 2024-06-11T14:03:22.868 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:03:22.868 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:03:23.198 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:03:23.199 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:03:23.199 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "11/23 daemons upgraded", 2024-06-11T14:03:23.199 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:03:23.199 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:03:23.199 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:03:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:23 smithi069 ceph-mon[66891]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:23 smithi069 ceph-mon[66891]: pgmap v142: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 257 B/s rd, 0 op/s 2024-06-11T14:03:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:23 smithi069 ceph-mon[66891]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:23 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3609547771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:23 smithi135 ceph-mon[53782]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:23 smithi135 ceph-mon[53782]: pgmap v142: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 257 B/s rd, 0 op/s 2024-06-11T14:03:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:23 smithi135 ceph-mon[53782]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:23 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3609547771' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:23.600 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:03:24.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:24 smithi069 ceph-mon[66891]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:24.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:24 smithi069 ceph-mon[66891]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:24.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:24 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3786584315' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:03:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:24 smithi135 ceph-mon[53782]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:24 smithi135 ceph-mon[53782]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:24 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3786584315' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:03:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:25 smithi069 ceph-mon[66891]: pgmap v143: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 241 B/s rd, 0 op/s 2024-06-11T14:03:25.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:25 smithi135 ceph-mon[53782]: pgmap v143: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 241 B/s rd, 0 op/s 2024-06-11T14:03:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:27 smithi069 ceph-mon[66891]: pgmap v144: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:03:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:27 smithi135 ceph-mon[53782]: pgmap v144: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:03:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:29 smithi069 ceph-mon[66891]: pgmap v145: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 178 B/s rd, 0 op/s 2024-06-11T14:03:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:29 smithi135 ceph-mon[53782]: pgmap v145: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 178 B/s rd, 0 op/s 2024-06-11T14:03:30.413 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:30 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:30.413 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:30 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:30 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:30 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:31 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:31 smithi069 ceph-mon[66891]: Upgrade: osd.5 is safe to restart 2024-06-11T14:03:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:31 smithi069 ceph-mon[66891]: pgmap v146: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:03:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:31 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:31 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-11T14:03:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:31 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:03:31.503 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:31 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-06-11T14:03:31.503 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:31 smithi135 ceph-mon[53782]: Upgrade: osd.5 is safe to restart 2024-06-11T14:03:31.504 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:31 smithi135 ceph-mon[53782]: pgmap v146: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:03:31.504 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:31 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:31.504 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:31 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-06-11T14:03:31.504 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:31 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:03:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:32 smithi069 ceph-mon[66891]: Upgrade: Updating osd.5 2024-06-11T14:03:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:32 smithi069 ceph-mon[66891]: Deploying daemon osd.5 on smithi135 2024-06-11T14:03:32.518 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:32 smithi135 ceph-mon[53782]: Upgrade: Updating osd.5 2024-06-11T14:03:32.518 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:32 smithi135 ceph-mon[53782]: Deploying daemon osd.5 on smithi135 2024-06-11T14:03:33.476 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:33 smithi135 ceph-mon[53782]: pgmap v147: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:33 smithi069 ceph-mon[66891]: pgmap v147: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:34.422 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:34 smithi135 systemd[1]: Stopping Ceph osd.5 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:03:34.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:34 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T14:03:34.422+0000 7f85d5882700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:03:34.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:34 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T14:03:34.422+0000 7f85d5882700 -1 osd.5 109 *** Got signal Terminated *** 2024-06-11T14:03:34.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:34 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[34680]: 2024-06-11T14:03:34.422+0000 7f85d5882700 -1 osd.5 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:03:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:35 smithi069 ceph-mon[66891]: pgmap v148: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:35 smithi069 ceph-mon[66891]: osd.5 marked itself down and dead 2024-06-11T14:03:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:35 smithi135 ceph-mon[53782]: pgmap v148: 97 pgs: 97 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:03:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:35 smithi135 ceph-mon[53782]: osd.5 marked itself down and dead 2024-06-11T14:03:36.081 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:35 smithi135 podman[60391]: 2024-06-11 14:03:35.810718497 +0000 UTC m=+1.514737629 container died 0528d0439ea609f87a526b85be2b9ca0afd96073655bd5c90f8a0c4e3d8763d4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754, io.openshift.expose-services=, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-06-11T14:03:36.439 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:36 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:03:36.439 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:36 smithi135 ceph-mon[53782]: osdmap e110: 8 total, 7 up, 8 in 2024-06-11T14:03:36.440 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:36 smithi135 podman[60391]: 2024-06-11 14:03:36.239023358 +0000 UTC m=+1.943042486 container cleanup 0528d0439ea609f87a526b85be2b9ca0afd96073655bd5c90f8a0c4e3d8763d4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, release=754, vendor=Red Hat, Inc., name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, version=8) 2024-06-11T14:03:36.440 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:36 smithi135 bash[60391]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5 2024-06-11T14:03:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:36 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:03:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:36 smithi069 ceph-mon[66891]: osdmap e110: 8 total, 7 up, 8 in 2024-06-11T14:03:36.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:36 smithi135 podman[60407]: 2024-06-11 14:03:36.440673189 +0000 UTC m=+0.633404715 container remove 0528d0439ea609f87a526b85be2b9ca0afd96073655bd5c90f8a0c4e3d8763d4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, RELEASE=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux ) 2024-06-11T14:03:37.159 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:36 smithi135 podman[60499]: 2024-06-11 14:03:36.759622514 +0000 UTC m=+0.017162579 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:03:37.159 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:36 smithi135 podman[60499]: 2024-06-11 14:03:36.866337364 +0000 UTC m=+0.123877403 container create d56d9af822247c18df22fecfeb1020bd0aa0095a0b696f3f9492e5ae75380dd9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:03:37.159 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 podman[60499]: 2024-06-11 14:03:37.077065844 +0000 UTC m=+0.334605884 container init d56d9af822247c18df22fecfeb1020bd0aa0095a0b696f3f9492e5ae75380dd9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528) 2024-06-11T14:03:37.159 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 podman[60499]: 2024-06-11 14:03:37.080887888 +0000 UTC m=+0.338427931 container start d56d9af822247c18df22fecfeb1020bd0aa0095a0b696f3f9492e5ae75380dd9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3) 2024-06-11T14:03:37.159 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 podman[60499]: 2024-06-11 14:03:37.160165964 +0000 UTC m=+0.417706007 container attach d56d9af822247c18df22fecfeb1020bd0aa0095a0b696f3f9492e5ae75380dd9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-06-11T14:03:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:37 smithi069 ceph-mon[66891]: pgmap v150: 97 pgs: 12 stale+active+clean, 85 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:03:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:37 smithi069 ceph-mon[66891]: osdmap e111: 8 total, 7 up, 8 in 2024-06-11T14:03:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:37 smithi135 ceph-mon[53782]: pgmap v150: 97 pgs: 12 stale+active+clean, 85 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:03:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:37 smithi135 ceph-mon[53782]: osdmap e111: 8 total, 7 up, 8 in 2024-06-11T14:03:37.547 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 podman[60499]: 2024-06-11 14:03:37.334527358 +0000 UTC m=+0.592067406 container died d56d9af822247c18df22fecfeb1020bd0aa0095a0b696f3f9492e5ae75380dd9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, GIT_CLEAN=True, org.label-schema.build-date=20240528, ceph=True, org.label-schema.vendor=CentOS) 2024-06-11T14:03:37.906 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 podman[60522]: 2024-06-11 14:03:37.90540597 +0000 UTC m=+0.565014438 container remove d56d9af822247c18df22fecfeb1020bd0aa0095a0b696f3f9492e5ae75380dd9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-06-11T14:03:38.260 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.5.service: Deactivated successfully. 2024-06-11T14:03:38.260 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 systemd[1]: Stopped Ceph osd.5 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:03:38.260 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:37 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.5.service: Consumed 4min 17.830s CPU time. 2024-06-11T14:03:38.260 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 systemd[1]: Starting Ceph osd.5 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:03:38.510 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:38 smithi135 ceph-mon[53782]: pgmap v152: 97 pgs: 12 stale+active+clean, 85 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-06-11T14:03:38.510 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 podman[60623]: 2024-06-11 14:03:38.242942556 +0000 UTC m=+0.016899534 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:03:38.510 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 podman[60623]: 2024-06-11 14:03:38.381644783 +0000 UTC m=+0.155601787 container create a2b73c2d8b2ac6bfaaa83f597fba007cdf581748291db5f43d7547fe5d5e1282 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-11T14:03:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:38 smithi069 ceph-mon[66891]: pgmap v152: 97 pgs: 12 stale+active+clean, 85 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-06-11T14:03:38.978 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 podman[60623]: 2024-06-11 14:03:38.60905363 +0000 UTC m=+0.383010592 container init a2b73c2d8b2ac6bfaaa83f597fba007cdf581748291db5f43d7547fe5d5e1282 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, io.buildah.version=1.35.3) 2024-06-11T14:03:38.978 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 podman[60623]: 2024-06-11 14:03:38.612651695 +0000 UTC m=+0.386608656 container start a2b73c2d8b2ac6bfaaa83f597fba007cdf581748291db5f43d7547fe5d5e1282 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3) 2024-06-11T14:03:38.979 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 podman[60623]: 2024-06-11 14:03:38.708661767 +0000 UTC m=+0.482618743 container attach a2b73c2d8b2ac6bfaaa83f597fba007cdf581748291db5f43d7547fe5d5e1282 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_CLEAN=True, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-06-11T14:03:39.297 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-11T14:03:39.297 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 bash[60623]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:38 smithi135 bash[60623]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 bash[60623]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 bash[60623]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 bash[60623]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 bash[60623]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate[60638]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 bash[60623]: --> ceph-volume raw activate successful for osd ID: 5 2024-06-11T14:03:39.298 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 podman[60623]: 2024-06-11 14:03:39.033276251 +0000 UTC m=+0.807233212 container died a2b73c2d8b2ac6bfaaa83f597fba007cdf581748291db5f43d7547fe5d5e1282 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:03:40.023 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 podman[60760]: 2024-06-11 14:03:39.679020502 +0000 UTC m=+0.639424377 container remove a2b73c2d8b2ac6bfaaa83f597fba007cdf581748291db5f43d7547fe5d5e1282 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-06-11T14:03:40.023 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 podman[60822]: 2024-06-11 14:03:39.793571768 +0000 UTC m=+0.017025927 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:03:40.023 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:39 smithi135 podman[60822]: 2024-06-11 14:03:39.913291113 +0000 UTC m=+0.136745258 container create 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:03:40.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:40 smithi069 ceph-mon[66891]: pgmap v153: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:40.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:40 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 46/426 objects degraded (10.798%), 14 pgs degraded (PG_DEGRADED) 2024-06-11T14:03:40.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:40 smithi135 ceph-mon[53782]: pgmap v153: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:40.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:40 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 46/426 objects degraded (10.798%), 14 pgs degraded (PG_DEGRADED) 2024-06-11T14:03:40.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:40 smithi135 podman[60822]: 2024-06-11 14:03:40.132498401 +0000 UTC m=+0.355952548 container init 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:03:40.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:40 smithi135 podman[60822]: 2024-06-11 14:03:40.135816958 +0000 UTC m=+0.359271104 container start 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-06-11T14:03:40.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:40 smithi135 ceph-osd[60841]: -- 172.21.15.135:0/600353710 <== mon.1 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5621520dc1e0 con 0x5621520b8000 2024-06-11T14:03:40.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:40 smithi135 bash[60822]: 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 2024-06-11T14:03:40.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:40 smithi135 systemd[1]: Started Ceph osd.5 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:03:41.016 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:40 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:03:40.677+0000 7f21cbd5b640 -1 Falling back to public interface 2024-06-11T14:03:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:41.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:42 smithi135 ceph-mon[53782]: pgmap v154: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:42.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:42 smithi069 ceph-mon[66891]: pgmap v154: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:45.198 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:44 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.198 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:44 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.198 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:44 smithi135 ceph-mon[53782]: pgmap v155: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 229 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:45.198 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:44 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.198 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:44 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:44 smithi069 ceph-mon[66891]: pgmap v155: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 229 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:45.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:44 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:46.538 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:46 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:03:46.395+0000 7f21cbd5b640 -1 osd.5 0 read_superblock omap replica is missing. 2024-06-11T14:03:47.447 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:47 smithi135 ceph-mon[53782]: pgmap v156: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:47 smithi069 ceph-mon[66891]: pgmap v156: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:47.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:47 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:03:47.446+0000 7f21cbd5b640 -1 osd.5 109 log_to_monitors true 2024-06-11T14:03:48.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='osd.5 [v2:172.21.15.135:6808/4218351302,v1:172.21.15.135:6809/4218351302]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T14:03:48.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T14:03:48.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:48.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:48 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:03:48.547 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:03:48.209+0000 7f21c3b06640 -1 osd.5 109 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:03:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='osd.5 [v2:172.21.15.135:6808/4218351302,v1:172.21.15.135:6809/4218351302]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T14:03:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-06-11T14:03:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:48.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:48 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: pgmap v157: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: osdmap e112: 8 total, 7 up, 8 in 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: from='osd.5 [v2:172.21.15.135:6808/4218351302,v1:172.21.15.135:6809/4218351302]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:49.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:03:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-11T14:03:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: pgmap v157: 97 pgs: 17 active+undersized, 14 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s; 46/426 objects degraded (10.798%) 2024-06-11T14:03:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-06-11T14:03:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: osdmap e112: 8 total, 7 up, 8 in 2024-06-11T14:03:49.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: from='osd.5 [v2:172.21.15.135:6808/4218351302,v1:172.21.15.135:6809/4218351302]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:49.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:03:49.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:03:49.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:03:50.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:50 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:03:50.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:50 smithi069 ceph-mon[66891]: osd.5 [v2:172.21.15.135:6808/4218351302,v1:172.21.15.135:6809/4218351302] boot 2024-06-11T14:03:50.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:50 smithi069 ceph-mon[66891]: osdmap e113: 8 total, 8 up, 8 in 2024-06-11T14:03:50.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:50 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T14:03:50.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:50 smithi069 ceph-mon[66891]: osdmap e114: 8 total, 8 up, 8 in 2024-06-11T14:03:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:50 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:03:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:50 smithi135 ceph-mon[53782]: osd.5 [v2:172.21.15.135:6808/4218351302,v1:172.21.15.135:6809/4218351302] boot 2024-06-11T14:03:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:50 smithi135 ceph-mon[53782]: osdmap e113: 8 total, 8 up, 8 in 2024-06-11T14:03:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:50 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-06-11T14:03:50.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:50 smithi135 ceph-mon[53782]: osdmap e114: 8 total, 8 up, 8 in 2024-06-11T14:03:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:51 smithi069 ceph-mon[66891]: pgmap v160: 97 pgs: 18 peering, 7 active+undersized, 6 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21/426 objects degraded (4.930%) 2024-06-11T14:03:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:51 smithi069 ceph-mon[66891]: Health check update: Degraded data redundancy: 21/426 objects degraded (4.930%), 6 pgs degraded (PG_DEGRADED) 2024-06-11T14:03:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:51 smithi135 ceph-mon[53782]: pgmap v160: 97 pgs: 18 peering, 7 active+undersized, 6 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21/426 objects degraded (4.930%) 2024-06-11T14:03:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:51 smithi135 ceph-mon[53782]: Health check update: Degraded data redundancy: 21/426 objects degraded (4.930%), 6 pgs degraded (PG_DEGRADED) 2024-06-11T14:03:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:53 smithi069 ceph-mon[66891]: pgmap v162: 97 pgs: 18 peering, 7 active+undersized, 6 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21/426 objects degraded (4.930%) 2024-06-11T14:03:53.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:53 smithi135 ceph-mon[53782]: pgmap v162: 97 pgs: 18 peering, 7 active+undersized, 6 active+undersized+degraded, 66 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21/426 objects degraded (4.930%) 2024-06-11T14:03:53.958 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:03:54.608 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (5m) 77s ago 27m 16.1M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (3m) 77s ago 27m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (3m) 10s ago 26m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (5m) 77s ago 27m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (21m) 77s ago 21m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (21m) 10s ago 21m 19.6M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (5m) 77s ago 28m 502M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (4m) 10s ago 26m 440M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (4m) 77s ago 28m 47.8M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (4m) 10s ago 26m 40.7M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (21m) 77s ago 21m 298M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:03:54.609 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (21m) 10s ago 21m 87.7M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (5m) 77s ago 27m 8996k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (5m) 10s ago 26m 15.0M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (3m) 77s ago 26m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (2m) 77s ago 25m 181M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (117s) 77s ago 25m 162M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (82s) 77s ago 24m 14.4M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (47s) 10s ago 24m 226M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (14s) 10s ago 23m 11.1M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (23m) 10s ago 23m 455M 2704M 17.2.0 e1d6a67b021e 2e1b7dcc548d 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (23m) 10s ago 23m 688M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:03:54.610 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (4m) 77s ago 26m 76.4M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:03:54.998 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 6 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:03:54.999 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:03:55.000 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:03:55.000 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:03:55.000 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-06-11T14:03:55.000 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 10 2024-06-11T14:03:55.000 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:03:55.000 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:03:55.340 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:55 smithi069 ceph-mon[66891]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:55.340 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:55 smithi069 ceph-mon[66891]: pgmap v163: 97 pgs: 18 peering, 1 active+undersized+degraded, 78 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/426 objects degraded (0.704%) 2024-06-11T14:03:55.340 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:55 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/4243271829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:55.340 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:03:55.341 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:03:55.341 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:03:55.341 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:03:55.341 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:03:55.341 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:03:55.341 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:03:55.342 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:03:55.342 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:03:55.342 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "12/23 daemons upgraded", 2024-06-11T14:03:55.342 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:03:55.342 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:03:55.343 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:03:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:55 smithi135 ceph-mon[53782]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:55 smithi135 ceph-mon[53782]: pgmap v163: 97 pgs: 18 peering, 1 active+undersized+degraded, 78 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/426 objects degraded (0.704%) 2024-06-11T14:03:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:55 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/4243271829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:03:55.732 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN Degraded data redundancy: 3/426 objects degraded (0.704%), 1 pg degraded 2024-06-11T14:03:55.732 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 3/426 objects degraded (0.704%), 1 pg degraded 2024-06-11T14:03:55.732 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-06-11T14:03:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:56 smithi069 ceph-mon[66891]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:56 smithi069 ceph-mon[66891]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:56 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3925033742' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:03:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:56 smithi069 ceph-mon[66891]: Health check update: Degraded data redundancy: 3/426 objects degraded (0.704%), 1 pg degraded (PG_DEGRADED) 2024-06-11T14:03:56.531 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:56 smithi135 ceph-mon[53782]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:56.531 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:56 smithi135 ceph-mon[53782]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:56.531 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:56 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3925033742' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:03:56.531 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:56 smithi135 ceph-mon[53782]: Health check update: Degraded data redundancy: 3/426 objects degraded (0.704%), 1 pg degraded (PG_DEGRADED) 2024-06-11T14:03:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:57 smithi069 ceph-mon[66891]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:57 smithi069 ceph-mon[66891]: pgmap v164: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 257 B/s rd, 0 op/s 2024-06-11T14:03:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:57 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/426 objects degraded (0.704%), 1 pg degraded) 2024-06-11T14:03:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:57 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:03:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:57 smithi135 ceph-mon[53782]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:03:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:57 smithi135 ceph-mon[53782]: pgmap v164: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 257 B/s rd, 0 op/s 2024-06-11T14:03:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:57 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/426 objects degraded (0.704%), 1 pg degraded) 2024-06-11T14:03:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:57 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:03:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:03:58 smithi069 ceph-mon[66891]: pgmap v165: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 229 B/s rd, 0 op/s 2024-06-11T14:03:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:03:58 smithi135 ceph-mon[53782]: pgmap v165: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 229 B/s rd, 0 op/s 2024-06-11T14:04:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:01 smithi069 ceph-mon[66891]: pgmap v166: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:04:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:01 smithi135 ceph-mon[53782]: pgmap v166: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:04:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:03 smithi069 ceph-mon[66891]: pgmap v167: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-06-11T14:04:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:04:03.512 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:03 smithi135 ceph-mon[53782]: pgmap v167: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-06-11T14:04:03.512 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:04:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:04 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:04:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:04 smithi069 ceph-mon[66891]: Upgrade: osd.6 is safe to restart 2024-06-11T14:04:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:04 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-06-11T14:04:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:04 smithi135 ceph-mon[53782]: Upgrade: osd.6 is safe to restart 2024-06-11T14:04:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:05 smithi069 ceph-mon[66891]: pgmap v168: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-11T14:04:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:05.505 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:05 smithi135 ceph-mon[53782]: pgmap v168: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:05.505 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:05.505 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:05.505 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-06-11T14:04:05.505 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:06.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:06 smithi135 ceph-mon[53782]: Upgrade: Updating osd.6 2024-06-11T14:04:06.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:06 smithi135 ceph-mon[53782]: Deploying daemon osd.6 on smithi135 2024-06-11T14:04:06.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:06 smithi069 ceph-mon[66891]: Upgrade: Updating osd.6 2024-06-11T14:04:06.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:06 smithi069 ceph-mon[66891]: Deploying daemon osd.6 on smithi135 2024-06-11T14:04:07.461 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:07 smithi135 ceph-mon[53782]: pgmap v169: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:04:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:07 smithi069 ceph-mon[66891]: pgmap v169: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:04:07.720 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:07 smithi135 systemd[1]: Stopping Ceph osd.6 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:04:08.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[37639]: 2024-06-11T14:04:07.768+0000 7eff214c2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:04:08.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[37639]: 2024-06-11T14:04:07.768+0000 7eff214c2700 -1 osd.6 114 *** Got signal Terminated *** 2024-06-11T14:04:08.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[37639]: 2024-06-11T14:04:07.768+0000 7eff214c2700 -1 osd.6 114 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:04:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:08 smithi069 ceph-mon[66891]: osd.6 marked itself down and dead 2024-06-11T14:04:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:08 smithi135 ceph-mon[53782]: osd.6 marked itself down and dead 2024-06-11T14:04:08.975 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:08 smithi135 podman[62645]: 2024-06-11 14:04:08.684375047 +0000 UTC m=+1.036322114 container died 2e1b7dcc548d39d86f62352fda61bcfd01fd81615ed1f19aaa288f6ccb1000b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.openshift.tags=base centos centos-stream, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8) 2024-06-11T14:04:09.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:09 smithi135 ceph-mon[53782]: pgmap v170: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:09.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:09 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:04:09.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:09 smithi135 ceph-mon[53782]: osdmap e115: 8 total, 7 up, 8 in 2024-06-11T14:04:09.298 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:09 smithi135 podman[62645]: 2024-06-11 14:04:09.150620744 +0000 UTC m=+1.502567870 container cleanup 2e1b7dcc548d39d86f62352fda61bcfd01fd81615ed1f19aaa288f6ccb1000b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, distribution-scope=public, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-06-11T14:04:09.298 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:09 smithi135 bash[62645]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6 2024-06-11T14:04:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:09 smithi069 ceph-mon[66891]: pgmap v170: 97 pgs: 97 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:09 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:04:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:09 smithi069 ceph-mon[66891]: osdmap e115: 8 total, 7 up, 8 in 2024-06-11T14:04:09.787 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:09 smithi135 podman[62672]: 2024-06-11 14:04:09.452252073 +0000 UTC m=+0.762049501 container remove 2e1b7dcc548d39d86f62352fda61bcfd01fd81615ed1f19aaa288f6ccb1000b9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, io.k8s.display-name=CentOS Stream 8, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, release=754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-06-11T14:04:10.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:09 smithi135 podman[62761]: 2024-06-11 14:04:09.690727395 +0000 UTC m=+0.016882267 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:04:10.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:09 smithi135 podman[62761]: 2024-06-11 14:04:09.803136416 +0000 UTC m=+0.129291271 container create e2774b8d675b2ec6c6f444f3c88dbaac6e1423a1967b0a964875b2d8872a55e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240528, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:04:10.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 podman[62761]: 2024-06-11 14:04:10.022467985 +0000 UTC m=+0.348622841 container init e2774b8d675b2ec6c6f444f3c88dbaac6e1423a1967b0a964875b2d8872a55e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528) 2024-06-11T14:04:10.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 podman[62761]: 2024-06-11 14:04:10.025644115 +0000 UTC m=+0.351798966 container start e2774b8d675b2ec6c6f444f3c88dbaac6e1423a1967b0a964875b2d8872a55e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-06-11T14:04:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:10 smithi069 ceph-mon[66891]: osdmap e116: 8 total, 7 up, 8 in 2024-06-11T14:04:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:10 smithi069 ceph-mon[66891]: pgmap v173: 97 pgs: 16 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 67 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36/426 objects degraded (8.451%) 2024-06-11T14:04:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:10 smithi135 ceph-mon[53782]: osdmap e116: 8 total, 7 up, 8 in 2024-06-11T14:04:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:10 smithi135 ceph-mon[53782]: pgmap v173: 97 pgs: 16 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 67 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36/426 objects degraded (8.451%) 2024-06-11T14:04:10.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 podman[62761]: 2024-06-11 14:04:10.114059215 +0000 UTC m=+0.440214076 container attach e2774b8d675b2ec6c6f444f3c88dbaac6e1423a1967b0a964875b2d8872a55e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-06-11T14:04:10.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 podman[62761]: 2024-06-11 14:04:10.283419695 +0000 UTC m=+0.609574548 container died e2774b8d675b2ec6c6f444f3c88dbaac6e1423a1967b0a964875b2d8872a55e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, ceph=True, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:04:11.221 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 podman[62783]: 2024-06-11 14:04:10.901081205 +0000 UTC m=+0.611419772 container remove e2774b8d675b2ec6c6f444f3c88dbaac6e1423a1967b0a964875b2d8872a55e6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:04:11.222 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.6.service: Deactivated successfully. 2024-06-11T14:04:11.222 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 systemd[1]: Stopped Ceph osd.6 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:04:11.222 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:10 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.6.service: Consumed 3min 42.659s CPU time. 2024-06-11T14:04:11.222 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 systemd[1]: Starting Ceph osd.6 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:04:11.481 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 36/426 objects degraded (8.451%), 12 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:11.481 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 podman[62886]: 2024-06-11 14:04:11.335182871 +0000 UTC m=+0.107250881 container create d7bd28c682f37661fa80028eea60df7809363e8f7058bf8a3daa6352beba3a17 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD) 2024-06-11T14:04:11.481 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 podman[62886]: 2024-06-11 14:04:11.245265609 +0000 UTC m=+0.017333625 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:04:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:11 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 36/426 objects degraded (8.451%), 12 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:11.967 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 podman[62886]: 2024-06-11 14:04:11.604769481 +0000 UTC m=+0.376837500 container init d7bd28c682f37661fa80028eea60df7809363e8f7058bf8a3daa6352beba3a17 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-06-11T14:04:11.967 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 podman[62886]: 2024-06-11 14:04:11.608019197 +0000 UTC m=+0.380087214 container start d7bd28c682f37661fa80028eea60df7809363e8f7058bf8a3daa6352beba3a17 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-06-11T14:04:11.967 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 podman[62886]: 2024-06-11 14:04:11.687971224 +0000 UTC m=+0.460039257 container attach d7bd28c682f37661fa80028eea60df7809363e8f7058bf8a3daa6352beba3a17 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-11T14:04:12.226 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate[62902]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:11 smithi135 bash[62886]: --> ceph-volume raw activate successful for osd ID: 6 2024-06-11T14:04:12.227 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:12 smithi135 podman[62886]: 2024-06-11 14:04:12.022404014 +0000 UTC m=+0.794472041 container died d7bd28c682f37661fa80028eea60df7809363e8f7058bf8a3daa6352beba3a17 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.vendor=CentOS) 2024-06-11T14:04:12.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:12 smithi069 ceph-mon[66891]: pgmap v174: 97 pgs: 16 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 67 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36/426 objects degraded (8.451%) 2024-06-11T14:04:12.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:12 smithi135 ceph-mon[53782]: pgmap v174: 97 pgs: 16 active+undersized, 2 stale+active+clean, 12 active+undersized+degraded, 67 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 36/426 objects degraded (8.451%) 2024-06-11T14:04:13.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:12 smithi135 podman[63020]: 2024-06-11 14:04:12.674888476 +0000 UTC m=+0.641987368 container remove d7bd28c682f37661fa80028eea60df7809363e8f7058bf8a3daa6352beba3a17 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-activate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, org.label-schema.build-date=20240528) 2024-06-11T14:04:13.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:12 smithi135 podman[63081]: 2024-06-11 14:04:12.789507197 +0000 UTC m=+0.016803131 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:04:13.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:12 smithi135 podman[63081]: 2024-06-11 14:04:12.923191122 +0000 UTC m=+0.150487036 container create 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:04:13.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:13 smithi135 podman[63081]: 2024-06-11 14:04:13.186497678 +0000 UTC m=+0.413793593 container init 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:04:13.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:13 smithi135 podman[63081]: 2024-06-11 14:04:13.189687975 +0000 UTC m=+0.416983900 container start 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528) 2024-06-11T14:04:13.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:13 smithi135 ceph-osd[63101]: -- 172.21.15.135:0/3077901967 <== mon.1 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563fd2cea1e0 con 0x563fd1ee5c00 2024-06-11T14:04:13.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:13 smithi135 bash[63081]: 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 2024-06-11T14:04:13.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:13 smithi135 systemd[1]: Started Ceph osd.6 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:04:14.034 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:04:13.728+0000 7fdb3f065640 -1 Falling back to public interface 2024-06-11T14:04:14.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:14.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:14.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:14.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:14 smithi069 ceph-mon[66891]: pgmap v175: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:14.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:14.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:14.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:14.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:14 smithi135 ceph-mon[53782]: pgmap v175: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:16 smithi069 ceph-mon[66891]: Health check update: Degraded data redundancy: 47/426 objects degraded (11.033%), 15 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:16.508 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:16 smithi135 ceph-mon[53782]: Health check update: Degraded data redundancy: 47/426 objects degraded (11.033%), 15 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:17 smithi069 ceph-mon[66891]: pgmap v176: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:17 smithi135 ceph-mon[53782]: pgmap v176: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:18.518 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:18.518 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:18 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:18.518 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:18 smithi135 ceph-mon[53782]: pgmap v177: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:18.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:18.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:18 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:18.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:18 smithi069 ceph-mon[66891]: pgmap v177: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:19.088 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:04:18.835+0000 7fdb3f065640 -1 osd.6 0 read_superblock omap replica is missing. 2024-06-11T14:04:20.297 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:19 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:04:19.914+0000 7fdb3f065640 -1 osd.6 114 log_to_monitors true 2024-06-11T14:04:20.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:20.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:20.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:20 smithi135 ceph-mon[53782]: from='osd.6 [v2:172.21.15.135:6816/414347442,v1:172.21.15.135:6817/414347442]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T14:04:20.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:20 smithi135 ceph-mon[53782]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T14:04:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:20 smithi069 ceph-mon[66891]: from='osd.6 [v2:172.21.15.135:6816/414347442,v1:172.21.15.135:6817/414347442]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T14:04:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:20 smithi069 ceph-mon[66891]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-06-11T14:04:20.772 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:04:20 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:04:20.608+0000 7fdb32e08640 -1 osd.6 114 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:04:21.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-11T14:04:21.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: pgmap v178: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:21.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: osdmap e117: 8 total, 7 up, 8 in 2024-06-11T14:04:21.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='osd.6 [v2:172.21.15.135:6816/414347442,v1:172.21.15.135:6817/414347442]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:21.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='osd.6 ' entity='osd.6' 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:21.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:21 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: pgmap v178: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 47/426 objects degraded (11.033%) 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: osdmap e117: 8 total, 7 up, 8 in 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='osd.6 [v2:172.21.15.135:6816/414347442,v1:172.21.15.135:6817/414347442]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='osd.6 ' entity='osd.6' 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:21.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:21 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:22 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:22 smithi069 ceph-mon[66891]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-11T14:04:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:22 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:04:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:22 smithi069 ceph-mon[66891]: osd.6 [v2:172.21.15.135:6816/414347442,v1:172.21.15.135:6817/414347442] boot 2024-06-11T14:04:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:22 smithi069 ceph-mon[66891]: osdmap e118: 8 total, 8 up, 8 in 2024-06-11T14:04:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:22 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T14:04:22.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:22 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:22.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:22 smithi135 ceph-mon[53782]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-06-11T14:04:22.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:22 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:04:22.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:22 smithi135 ceph-mon[53782]: osd.6 [v2:172.21.15.135:6816/414347442,v1:172.21.15.135:6817/414347442] boot 2024-06-11T14:04:22.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:22 smithi135 ceph-mon[53782]: osdmap e118: 8 total, 8 up, 8 in 2024-06-11T14:04:22.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:22 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-06-11T14:04:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:23 smithi069 ceph-mon[66891]: pgmap v181: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 47/426 objects degraded (11.033%) 2024-06-11T14:04:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:23 smithi069 ceph-mon[66891]: osdmap e119: 8 total, 8 up, 8 in 2024-06-11T14:04:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:23 smithi135 ceph-mon[53782]: pgmap v181: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 47/426 objects degraded (11.033%) 2024-06-11T14:04:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:23 smithi135 ceph-mon[53782]: osdmap e119: 8 total, 8 up, 8 in 2024-06-11T14:04:24.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:24 smithi069 ceph-mon[66891]: Health check update: Degraded data redundancy: 41/426 objects degraded (9.624%), 13 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:24 smithi135 ceph-mon[53782]: Health check update: Degraded data redundancy: 41/426 objects degraded (9.624%), 13 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:25 smithi069 ceph-mon[66891]: pgmap v183: 97 pgs: 17 active+undersized, 13 active+undersized+degraded, 67 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:04:25.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:25 smithi135 ceph-mon[53782]: pgmap v183: 97 pgs: 17 active+undersized, 13 active+undersized+degraded, 67 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41/426 objects degraded (9.624%) 2024-06-11T14:04:26.089 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:04:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:26 smithi069 ceph-mon[66891]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:26 smithi069 ceph-mon[66891]: pgmap v184: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:04:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:26 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/426 objects degraded (9.624%), 13 pgs degraded) 2024-06-11T14:04:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:26 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:04:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:26 smithi135 ceph-mon[53782]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:26 smithi135 ceph-mon[53782]: pgmap v184: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:04:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:26 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/426 objects degraded (9.624%), 13 pgs degraded) 2024-06-11T14:04:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:26 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:04:26.737 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:04:26.737 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (6m) 109s ago 28m 16.1M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:04:26.737 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (4m) 109s ago 28m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (3m) 9s ago 27m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (5m) 109s ago 27m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (22m) 109s ago 22m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (22m) 9s ago 22m 19.7M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (5m) 109s ago 29m 502M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (5m) 9s ago 27m 440M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (5m) 109s ago 29m 47.8M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (4m) 9s ago 27m 42.1M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (21m) 109s ago 21m 298M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (21m) 9s ago 21m 87.7M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (5m) 109s ago 27m 8996k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:04:26.738 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (5m) 9s ago 27m 15.5M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (3m) 109s ago 26m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (3m) 109s ago 26m 181M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (2m) 109s ago 25m 162M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (115s) 109s ago 25m 14.4M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (79s) 9s ago 24m 226M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (46s) 9s ago 24m 201M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (13s) 9s ago 24m 11.1M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (23m) 9s ago 23m 687M 2704M 17.2.0 e1d6a67b021e 578d12f9cc24 2024-06-11T14:04:26.739 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (5m) 109s ago 26m 76.4M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-11T14:04:27.129 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 7 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 11 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:04:27.130 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:04:27.465 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr" 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "13/23 daemons upgraded", 2024-06-11T14:04:27.466 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading osd daemons", 2024-06-11T14:04:27.467 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:04:27.467 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:04:27.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:27 smithi069 ceph-mon[66891]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:27.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:27 smithi069 ceph-mon[66891]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:27.467 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:27 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1775804651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:27 smithi135 ceph-mon[53782]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:27 smithi135 ceph-mon[53782]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:27 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1775804651' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:27.858 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:04:28.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:28 smithi135 ceph-mon[53782]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:28.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:28 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1103305131' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:04:28.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:28 smithi135 ceph-mon[53782]: pgmap v185: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:04:28.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:28 smithi069 ceph-mon[66891]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:28.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:28 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1103305131' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:04:28.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:28 smithi069 ceph-mon[66891]: pgmap v185: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:04:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:31 smithi069 ceph-mon[66891]: pgmap v186: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 239 B/s rd, 0 op/s 2024-06-11T14:04:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:31 smithi135 ceph-mon[53782]: pgmap v186: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 239 B/s rd, 0 op/s 2024-06-11T14:04:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:33 smithi069 ceph-mon[66891]: pgmap v187: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:04:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:33 smithi135 ceph-mon[53782]: pgmap v187: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:04:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:35 smithi069 ceph-mon[66891]: pgmap v188: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 177 B/s rd, 0 op/s 2024-06-11T14:04:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:35 smithi135 ceph-mon[53782]: pgmap v188: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 177 B/s rd, 0 op/s 2024-06-11T14:04:35.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:36.478 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:37.280 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:37 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:37.281 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:37 smithi135 ceph-mon[53782]: Upgrade: osd.7 is safe to restart 2024-06-11T14:04:37.281 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:37 smithi135 ceph-mon[53782]: pgmap v189: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:04:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:37 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-06-11T14:04:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:37 smithi069 ceph-mon[66891]: Upgrade: osd.7 is safe to restart 2024-06-11T14:04:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:37 smithi069 ceph-mon[66891]: pgmap v189: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:04:38.635 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:38 smithi135 ceph-mon[53782]: Upgrade: Updating osd.7 2024-06-11T14:04:38.635 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:38.635 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-11T14:04:38.635 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:38.635 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:38 smithi135 ceph-mon[53782]: Deploying daemon osd.7 on smithi135 2024-06-11T14:04:38.635 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:38 smithi135 ceph-mon[53782]: pgmap v190: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:38 smithi069 ceph-mon[66891]: Upgrade: Updating osd.7 2024-06-11T14:04:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-06-11T14:04:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:38 smithi069 ceph-mon[66891]: Deploying daemon osd.7 on smithi135 2024-06-11T14:04:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:38 smithi069 ceph-mon[66891]: pgmap v190: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:40.614 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:40 smithi135 systemd[1]: Stopping Ceph osd.7 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:04:41.047 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:40 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[40710]: 2024-06-11T14:04:40.614+0000 7fadd7cc6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:04:41.047 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:40 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[40710]: 2024-06-11T14:04:40.614+0000 7fadd7cc6700 -1 osd.7 119 *** Got signal Terminated *** 2024-06-11T14:04:41.047 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:40 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[40710]: 2024-06-11T14:04:40.614+0000 7fadd7cc6700 -1 osd.7 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:04:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:41 smithi069 ceph-mon[66891]: pgmap v191: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:41 smithi069 ceph-mon[66891]: osd.7 marked itself down and dead 2024-06-11T14:04:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:41 smithi135 ceph-mon[53782]: pgmap v191: 97 pgs: 97 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:04:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:41 smithi135 ceph-mon[53782]: osd.7 marked itself down and dead 2024-06-11T14:04:42.114 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:41 smithi135 podman[64883]: 2024-06-11 14:04:41.83073356 +0000 UTC m=+1.331537480 container died 578d12f9cc243fdbb9252fef718f88f88862a470e6d9d3379661fd50acc042ba (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, version=8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870) 2024-06-11T14:04:42.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:42 smithi069 ceph-mon[66891]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:04:42.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:42 smithi069 ceph-mon[66891]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-11T14:04:42.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:42 smithi069 ceph-mon[66891]: osdmap e120: 8 total, 7 up, 8 in 2024-06-11T14:04:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:42 smithi135 ceph-mon[53782]: Health check failed: 1 osds down (OSD_DOWN) 2024-06-11T14:04:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:42 smithi135 ceph-mon[53782]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-06-11T14:04:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:42 smithi135 ceph-mon[53782]: osdmap e120: 8 total, 7 up, 8 in 2024-06-11T14:04:42.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:42 smithi135 podman[64883]: 2024-06-11 14:04:42.31449931 +0000 UTC m=+1.815303219 container cleanup 578d12f9cc243fdbb9252fef718f88f88862a470e6d9d3379661fd50acc042ba (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, architecture=x86_64, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, maintainer=Guillaume Abrioux , distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-06-11T14:04:42.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:42 smithi135 bash[64883]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7 2024-06-11T14:04:42.986 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:42 smithi135 podman[64899]: 2024-06-11 14:04:42.641211353 +0000 UTC m=+0.809416317 container remove 578d12f9cc243fdbb9252fef718f88f88862a470e6d9d3379661fd50acc042ba (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, distribution-scope=public) 2024-06-11T14:04:43.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:43 smithi135 ceph-mon[53782]: pgmap v193: 97 pgs: 12 stale+active+clean, 85 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:04:43.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:43 smithi135 ceph-mon[53782]: osdmap e121: 8 total, 7 up, 8 in 2024-06-11T14:04:43.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:42 smithi135 podman[64990]: 2024-06-11 14:04:42.890622769 +0000 UTC m=+0.017205104 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:04:43.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:43 smithi135 podman[64990]: 2024-06-11 14:04:43.000384679 +0000 UTC m=+0.126967075 container create 2bfe2cf367dd9b1f68fbfb5cc704aaab35a67199b9fc44b2bdb145b010e47491 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:04:43.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:43 smithi135 podman[64990]: 2024-06-11 14:04:43.227997839 +0000 UTC m=+0.354580185 container init 2bfe2cf367dd9b1f68fbfb5cc704aaab35a67199b9fc44b2bdb145b010e47491 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-06-11T14:04:43.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:43 smithi135 podman[64990]: 2024-06-11 14:04:43.232351922 +0000 UTC m=+0.358934251 container start 2bfe2cf367dd9b1f68fbfb5cc704aaab35a67199b9fc44b2bdb145b010e47491 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:04:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:43 smithi069 ceph-mon[66891]: pgmap v193: 97 pgs: 12 stale+active+clean, 85 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-06-11T14:04:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:43 smithi069 ceph-mon[66891]: osdmap e121: 8 total, 7 up, 8 in 2024-06-11T14:04:43.780 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:43 smithi135 podman[64990]: 2024-06-11 14:04:43.319557155 +0000 UTC m=+0.446139501 container attach 2bfe2cf367dd9b1f68fbfb5cc704aaab35a67199b9fc44b2bdb145b010e47491 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3) 2024-06-11T14:04:43.780 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:43 smithi135 podman[64990]: 2024-06-11 14:04:43.489760253 +0000 UTC m=+0.616342581 container died 2bfe2cf367dd9b1f68fbfb5cc704aaab35a67199b9fc44b2bdb145b010e47491 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-06-11T14:04:44.539 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:44 smithi135 ceph-mon[53782]: pgmap v195: 97 pgs: 4 active+undersized, 11 stale+active+clean, 3 active+undersized+degraded, 79 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12/426 objects degraded (2.817%) 2024-06-11T14:04:44.539 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:44 smithi135 ceph-mon[53782]: Health check failed: Degraded data redundancy: 12/426 objects degraded (2.817%), 3 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:44.540 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 podman[65044]: 2024-06-11 14:04:44.106492283 +0000 UTC m=+0.611534810 container remove 2bfe2cf367dd9b1f68fbfb5cc704aaab35a67199b9fc44b2bdb145b010e47491 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3) 2024-06-11T14:04:44.540 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.7.service: Deactivated successfully. 2024-06-11T14:04:44.540 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 systemd[1]: Stopped Ceph osd.7 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:04:44.540 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 systemd[1]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.7.service: Consumed 5min 11.965s CPU time. 2024-06-11T14:04:44.540 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 systemd[1]: Starting Ceph osd.7 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:04:44.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:44 smithi069 ceph-mon[66891]: pgmap v195: 97 pgs: 4 active+undersized, 11 stale+active+clean, 3 active+undersized+degraded, 79 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12/426 objects degraded (2.817%) 2024-06-11T14:04:44.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:44 smithi069 ceph-mon[66891]: Health check failed: Degraded data redundancy: 12/426 objects degraded (2.817%), 3 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:44.797 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 podman[65144]: 2024-06-11 14:04:44.442930857 +0000 UTC m=+0.017355573 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:04:44.797 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 podman[65144]: 2024-06-11 14:04:44.540556961 +0000 UTC m=+0.114981663 container create e11aa77958214061b5e4d5c02156f4068e820d56f2e803b16a4b7e071b92005f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-06-11T14:04:44.797 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 podman[65144]: 2024-06-11 14:04:44.773422523 +0000 UTC m=+0.347847236 container init e11aa77958214061b5e4d5c02156f4068e820d56f2e803b16a4b7e071b92005f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-06-11T14:04:44.797 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 podman[65144]: 2024-06-11 14:04:44.777068567 +0000 UTC m=+0.351493271 container start e11aa77958214061b5e4d5c02156f4068e820d56f2e803b16a4b7e071b92005f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux ) 2024-06-11T14:04:45.143 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:44 smithi135 podman[65144]: 2024-06-11 14:04:44.885107067 +0000 UTC m=+0.459531788 container attach e11aa77958214061b5e4d5c02156f4068e820d56f2e803b16a4b7e071b92005f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:04:45.543 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-11T14:04:45.543 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-11T14:04:45.543 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-11T14:04:45.543 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-06-11T14:04:45.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-11T14:04:45.545 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-06-11T14:04:45.545 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate[65159]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-11T14:04:45.545 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 bash[65144]: --> ceph-volume raw activate successful for osd ID: 7 2024-06-11T14:04:45.545 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 podman[65144]: 2024-06-11 14:04:45.199144014 +0000 UTC m=+0.773568728 container died e11aa77958214061b5e4d5c02156f4068e820d56f2e803b16a4b7e071b92005f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3) 2024-06-11T14:04:46.208 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:45 smithi135 podman[65277]: 2024-06-11 14:04:45.864067553 +0000 UTC m=+0.659249220 container remove e11aa77958214061b5e4d5c02156f4068e820d56f2e803b16a4b7e071b92005f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:04:46.208 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 podman[65334]: 2024-06-11 14:04:45.97908772 +0000 UTC m=+0.017042484 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:04:46.209 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 podman[65334]: 2024-06-11 14:04:46.098123178 +0000 UTC m=+0.136077924 container create 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-06-11T14:04:46.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 podman[65334]: 2024-06-11 14:04:46.342159365 +0000 UTC m=+0.380114130 container init 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528) 2024-06-11T14:04:46.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 podman[65334]: 2024-06-11 14:04:46.345822241 +0000 UTC m=+0.383776995 container start 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, RELEASE=squid-3f51ea3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS) 2024-06-11T14:04:46.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 ceph-osd[65353]: -- 172.21.15.135:0/3095327477 <== mon.1 v2:172.21.15.135:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fb724ca1e0 con 0x55fb716c1c00 2024-06-11T14:04:46.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 bash[65334]: 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 2024-06-11T14:04:46.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 systemd[1]: Started Ceph osd.7 for 5368e33c-27f7-11ef-bc9b-c7b262605968. 2024-06-11T14:04:47.195 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:46 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:04:46.887+0000 7f0a92c6b640 -1 Falling back to public interface 2024-06-11T14:04:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:47 smithi069 ceph-mon[66891]: pgmap v196: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:47 smithi135 ceph-mon[53782]: pgmap v196: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:49 smithi069 ceph-mon[66891]: pgmap v197: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:49 smithi135 ceph-mon[53782]: pgmap v197: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:49.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:04:51.358 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.359 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.359 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:51 smithi135 ceph-mon[53782]: pgmap v198: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:51.359 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.359 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:51 smithi069 ceph-mon[66891]: pgmap v198: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:52.382 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:52 smithi135 ceph-mon[53782]: Health check update: Degraded data redundancy: 78/426 objects degraded (18.310%), 22 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:52.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:52 smithi069 ceph-mon[66891]: Health check update: Degraded data redundancy: 78/426 objects degraded (18.310%), 22 pgs degraded (PG_DEGRADED) 2024-06-11T14:04:53.384 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:53 smithi135 ceph-mon[53782]: pgmap v199: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:53.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:53 smithi069 ceph-mon[66891]: pgmap v199: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:53.794 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:53 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:04:53.603+0000 7f0a92c6b640 -1 osd.7 0 read_superblock omap replica is missing. 2024-06-11T14:04:55.047 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:54 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:04:54.743+0000 7f0a92c6b640 -1 osd.7 119 log_to_monitors true 2024-06-11T14:04:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: pgmap v200: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:55.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-11T14:04:55.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='osd.7 [v2:172.21.15.135:6824/3297038123,v1:172.21.15.135:6825/3297038123]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T14:04:55.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:55 smithi069 ceph-mon[66891]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T14:04:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: pgmap v200: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 78/426 objects degraded (18.310%) 2024-06-11T14:04:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-06-11T14:04:55.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='osd.7 [v2:172.21.15.135:6824/3297038123,v1:172.21.15.135:6825/3297038123]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T14:04:55.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-mon[53782]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-06-11T14:04:56.201 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:04:55 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:04:55.915+0000 7f0a86a0e640 -1 osd.7 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-06-11T14:04:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all osd 2024-06-11T14:04:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: Upgrade: Setting require_osd_release to 19 squid 2024-06-11T14:04:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-11T14:04:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-11T14:04:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-11T14:04:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: from='osd.7 [v2:172.21.15.135:6824/3297038123,v1:172.21.15.135:6825/3297038123]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:56.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: osdmap e122: 8 total, 7 up, 8 in 2024-06-11T14:04:56.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:56.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi069.nsgdqs"]}]: dispatch 2024-06-11T14:04:56.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:56 smithi069 ceph-mon[66891]: from='osd.7 ' entity='osd.7' 2024-06-11T14:04:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all osd 2024-06-11T14:04:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: Upgrade: Setting require_osd_release to 19 squid 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: from='osd.7 [v2:172.21.15.135:6824/3297038123,v1:172.21.15.135:6825/3297038123]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: osdmap e122: 8 total, 7 up, 8 in 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi135", "root=default"]}]: dispatch 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi069.nsgdqs"]}]: dispatch 2024-06-11T14:04:56.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:56 smithi135 ceph-mon[53782]: from='osd.7 ' entity='osd.7' 2024-06-11T14:04:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: Upgrade: It appears safe to stop mds.foofs.smithi069.nsgdqs 2024-06-11T14:04:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: pgmap v202: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 78/426 objects degraded (18.310%) 2024-06-11T14:04:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:04:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:57.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:04:57.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: osd.7 [v2:172.21.15.135:6824/3297038123,v1:172.21.15.135:6825/3297038123] boot 2024-06-11T14:04:57.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: osdmap e123: 8 total, 8 up, 8 in 2024-06-11T14:04:57.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:57 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T14:04:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: Upgrade: It appears safe to stop mds.foofs.smithi069.nsgdqs 2024-06-11T14:04:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: pgmap v202: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 78/426 objects degraded (18.310%) 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi069.nsgdqs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: osd.7 [v2:172.21.15.135:6824/3297038123,v1:172.21.15.135:6825/3297038123] boot 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: osdmap e123: 8 total, 8 up, 8 in 2024-06-11T14:04:57.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:57 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-06-11T14:04:58.227 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:04:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:58 smithi069 ceph-mon[66891]: Upgrade: Updating mds.foofs.smithi069.nsgdqs 2024-06-11T14:04:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:58 smithi069 ceph-mon[66891]: Deploying daemon mds.foofs.smithi069.nsgdqs on smithi069 2024-06-11T14:04:58.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:58 smithi069 ceph-mon[66891]: osdmap e124: 8 total, 8 up, 8 in 2024-06-11T14:04:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:58 smithi135 ceph-mon[53782]: Upgrade: Updating mds.foofs.smithi069.nsgdqs 2024-06-11T14:04:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:58 smithi135 ceph-mon[53782]: Deploying daemon mds.foofs.smithi069.nsgdqs on smithi069 2024-06-11T14:04:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:58 smithi135 ceph-mon[53782]: osdmap e124: 8 total, 8 up, 8 in 2024-06-11T14:04:58.926 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (6m) 2m ago 28m 16.1M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (4m) 2m ago 28m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (4m) 8s ago 27m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (6m) 2m ago 28m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (22m) 2m ago 22m 39.7M - 17.2.0 e1d6a67b021e 52597ad140c5 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (22m) 8s ago 22m 19.8M - 17.2.0 e1d6a67b021e 00b76d0b7f53 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (6m) 2m ago 29m 502M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (5m) 8s ago 27m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:04:58.927 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (5m) 2m ago 30m 47.8M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (5m) 8s ago 27m 43.2M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (22m) 2m ago 22m 298M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (22m) 8s ago 22m 87.8M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (6m) 2m ago 28m 8996k - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (6m) 8s ago 27m 15.5M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (4m) 2m ago 27m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (3m) 2m ago 26m 181M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (3m) 2m ago 26m 162M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (2m) 2m ago 25m 14.4M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:04:58.928 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (112s) 8s ago 25m 228M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:04:58.929 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (79s) 8s ago 25m 202M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:04:58.929 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (46s) 8s ago 24m 194M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:04:58.929 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (12s) 8s ago 24m 11.2M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:04:58.929 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (5m) 2m ago 27m 76.4M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:04:59.206 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:58 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[66887]: 2024-06-11T14:04:58.794+0000 7fbad2c56640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T14:04:59.338 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:04:59.338 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-06-11T14:04:59.339 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 12 2024-06-11T14:04:59.340 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:04:59.340 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: pgmap v205: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 78/426 objects degraded (18.310%) 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: osdmap e125: 8 total, 8 up, 8 in 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: Standby daemon mds.foofs.smithi135.wtrawd assigned to filesystem foofs as rank 0 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-11T14:04:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-11T14:04:59.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-11T14:04:59.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:04:59 smithi069 ceph-mon[66891]: fsmap foofs:1/1 {0=foofs.smithi135.wtrawd=up:replay} 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: pgmap v205: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 228 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 78/426 objects degraded (18.310%) 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: osdmap e125: 8 total, 8 up, 8 in 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: Standby daemon mds.foofs.smithi135.wtrawd assigned to filesystem foofs as rank 0 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-11T14:04:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-11T14:04:59.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:04:59 smithi135 ceph-mon[53782]: fsmap foofs:1/1 {0=foofs.smithi135.wtrawd=up:replay} 2024-06-11T14:04:59.677 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-06-11T14:04:59.678 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:04:59.679 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "14/23 daemons upgraded", 2024-06-11T14:04:59.679 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mds daemons", 2024-06-11T14:04:59.679 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:04:59.679 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:05:00.068 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available; Degraded data redundancy: 78/426 objects degraded (18.310%), 22 pgs degraded 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: fs foofs is degraded 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: have 0; want 1 more 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 78/426 objects degraded (18.310%), 22 pgs degraded 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-06-11T14:05:00.069 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-06-11T14:05:00.070 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-06-11T14:05:00.071 INFO:teuthology.orchestra.run.smithi069.stdout: pg 4.14 is active+undersized+degraded, acting [3,5] 2024-06-11T14:05:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:00 smithi069 ceph-mon[66891]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:00 smithi069 ceph-mon[66891]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:00 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/23894975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:00 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1970447380' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:05:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:00 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T14:05:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:00 smithi135 ceph-mon[53782]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:00 smithi135 ceph-mon[53782]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:00 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/23894975' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:00 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1970447380' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:05:00.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:00 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T14:05:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: pgmap v207: 97 pgs: 6 active+undersized, 10 active+undersized+degraded, 81 active+clean; 228 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 7 op/s; 34/426 objects degraded (7.981%) 2024-06-11T14:05:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: Health check update: Degraded data redundancy: 34/426 objects degraded (7.981%), 10 pgs degraded (PG_DEGRADED) 2024-06-11T14:05:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: reconnect by client.24351 172.21.15.135:0/4007316783 after 0.000999978 2024-06-11T14:05:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: reconnect by client.14558 172.21.15.69:0/1746076871 after 0.000999978 2024-06-11T14:05:01.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:reconnect 2024-06-11T14:05:01.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:01 smithi069 ceph-mon[66891]: fsmap foofs:1/1 {0=foofs.smithi135.wtrawd=up:reconnect} 2024-06-11T14:05:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: pgmap v207: 97 pgs: 6 active+undersized, 10 active+undersized+degraded, 81 active+clean; 228 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 7 op/s; 34/426 objects degraded (7.981%) 2024-06-11T14:05:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: Health check update: Degraded data redundancy: 34/426 objects degraded (7.981%), 10 pgs degraded (PG_DEGRADED) 2024-06-11T14:05:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: reconnect by client.24351 172.21.15.135:0/4007316783 after 0.000999978 2024-06-11T14:05:01.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: reconnect by client.14558 172.21.15.69:0/1746076871 after 0.000999978 2024-06-11T14:05:01.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:reconnect 2024-06-11T14:05:01.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:01 smithi135 ceph-mon[53782]: fsmap foofs:1/1 {0=foofs.smithi135.wtrawd=up:reconnect} 2024-06-11T14:05:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:02 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:rejoin 2024-06-11T14:05:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:02 smithi069 ceph-mon[66891]: fsmap foofs:1/1 {0=foofs.smithi135.wtrawd=up:rejoin} 2024-06-11T14:05:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:02 smithi069 ceph-mon[66891]: daemon mds.foofs.smithi135.wtrawd is now active in filesystem foofs as rank 0 2024-06-11T14:05:02.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:02 smithi069 ceph-mon[66891]: pgmap v208: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 9 op/s 2024-06-11T14:05:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:02 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:rejoin 2024-06-11T14:05:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:02 smithi135 ceph-mon[53782]: fsmap foofs:1/1 {0=foofs.smithi135.wtrawd=up:rejoin} 2024-06-11T14:05:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:02 smithi135 ceph-mon[53782]: daemon mds.foofs.smithi135.wtrawd is now active in filesystem foofs as rank 0 2024-06-11T14:05:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:02 smithi135 ceph-mon[53782]: pgmap v208: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 29 MiB/s rd, 9 op/s 2024-06-11T14:05:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:03 smithi069 ceph-mon[66891]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/426 objects degraded (7.981%), 10 pgs degraded) 2024-06-11T14:05:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:03 smithi069 ceph-mon[66891]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-11T14:05:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:03 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:active 2024-06-11T14:05:03.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:03 smithi069 ceph-mon[66891]: fsmap foofs:1 {0=foofs.smithi135.wtrawd=up:active} 2024-06-11T14:05:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:03 smithi135 ceph-mon[53782]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/426 objects degraded (7.981%), 10 pgs degraded) 2024-06-11T14:05:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:03 smithi135 ceph-mon[53782]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-11T14:05:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:03 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.135:6832/2439610600,v1:172.21.15.135:6833/2439610600] up:active 2024-06-11T14:05:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:03 smithi135 ceph-mon[53782]: fsmap foofs:1 {0=foofs.smithi135.wtrawd=up:active} 2024-06-11T14:05:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:05 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:05 smithi069 ceph-mon[66891]: pgmap v209: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 9 op/s 2024-06-11T14:05:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:05 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:05 smithi135 ceph-mon[53782]: pgmap v209: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 9 op/s 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:boot 2024-06-11T14:05:06.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: fsmap foofs:1 {0=foofs.smithi135.wtrawd=up:active} 1 up:standby 2024-06-11T14:05:06.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T14:05:06.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:06 smithi069 ceph-mon[66891]: pgmap v210: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 248 B/s wr, 13 op/s 2024-06-11T14:05:07.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:boot 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: fsmap foofs:1 {0=foofs.smithi135.wtrawd=up:active} 1 up:standby 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T14:05:07.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:06 smithi135 ceph-mon[53782]: pgmap v210: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 248 B/s wr, 13 op/s 2024-06-11T14:05:09.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:09 smithi069 ceph-mon[66891]: pgmap v211: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 10 op/s 2024-06-11T14:05:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:09 smithi135 ceph-mon[53782]: pgmap v211: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 10 op/s 2024-06-11T14:05:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:10.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:11 smithi069 ceph-mon[66891]: pgmap v212: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 180 B/s wr, 9 op/s 2024-06-11T14:05:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:11 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:11 smithi135 ceph-mon[53782]: pgmap v212: 97 pgs: 97 active+clean; 228 MiB data, 1004 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 180 B/s wr, 9 op/s 2024-06-11T14:05:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:11 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:12.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:12.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:12.393 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:12 smithi069 ceph-mon[66891]: pgmap v213: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-06-11T14:05:12.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:12.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:12.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:12 smithi135 ceph-mon[53782]: pgmap v213: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-06-11T14:05:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: pgmap v214: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-06-11T14:05:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:15 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi135.wtrawd"]}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: pgmap v214: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.501 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:15.502 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:15 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi135.wtrawd"]}]: dispatch 2024-06-11T14:05:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:16 smithi069 ceph-mon[66891]: Detected new or changed devices on smithi069 2024-06-11T14:05:16.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:16 smithi069 ceph-mon[66891]: Upgrade: It appears safe to stop mds.foofs.smithi135.wtrawd 2024-06-11T14:05:16.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:16 smithi135 ceph-mon[53782]: Detected new or changed devices on smithi069 2024-06-11T14:05:16.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:16 smithi135 ceph-mon[53782]: Upgrade: It appears safe to stop mds.foofs.smithi135.wtrawd 2024-06-11T14:05:17.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:17 smithi135 ceph-mon[53782]: pgmap v215: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 3.8 KiB/s rd, 7.3 KiB/s wr, 6 op/s 2024-06-11T14:05:17.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:17.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:05:17.298 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:17 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:17 smithi069 ceph-mon[66891]: pgmap v215: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 3.8 KiB/s rd, 7.3 KiB/s wr, 6 op/s 2024-06-11T14:05:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi135.wtrawd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-06-11T14:05:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:17 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:18.355 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:18 smithi135 ceph-mon[53782]: Upgrade: Updating mds.foofs.smithi135.wtrawd 2024-06-11T14:05:18.355 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:18 smithi135 ceph-mon[53782]: Deploying daemon mds.foofs.smithi135.wtrawd on smithi135 2024-06-11T14:05:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:18 smithi069 ceph-mon[66891]: Upgrade: Updating mds.foofs.smithi135.wtrawd 2024-06-11T14:05:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:18 smithi069 ceph-mon[66891]: Deploying daemon mds.foofs.smithi135.wtrawd on smithi135 2024-06-11T14:05:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:18 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[66887]: 2024-06-11T14:05:18.355+0000 7fbad2c56640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T14:05:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: pgmap v216: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-06-11T14:05:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-11T14:05:19.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T14:05:19.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: osdmap e126: 8 total, 8 up, 8 in 2024-06-11T14:05:19.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: Standby daemon mds.foofs.smithi069.nsgdqs assigned to filesystem foofs as rank 0 2024-06-11T14:05:19.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-11T14:05:19.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-11T14:05:19.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-11T14:05:19.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: fsmap foofs:1/1 {0=foofs.smithi069.nsgdqs=up:replay} 2024-06-11T14:05:19.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:19.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: pgmap v216: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-06-11T14:05:19.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-06-11T14:05:19.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-06-11T14:05:19.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: osdmap e126: 8 total, 8 up, 8 in 2024-06-11T14:05:19.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: Standby daemon mds.foofs.smithi069.nsgdqs assigned to filesystem foofs as rank 0 2024-06-11T14:05:19.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-06-11T14:05:19.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-06-11T14:05:19.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-06-11T14:05:19.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: fsmap foofs:1/1 {0=foofs.smithi069.nsgdqs=up:replay} 2024-06-11T14:05:19.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:19.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:20 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T14:05:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:20 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:reconnect 2024-06-11T14:05:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:20 smithi069 ceph-mon[66891]: fsmap foofs:1/1 {0=foofs.smithi069.nsgdqs=up:reconnect} 2024-06-11T14:05:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:20 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T14:05:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:20 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:reconnect 2024-06-11T14:05:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:20 smithi135 ceph-mon[53782]: fsmap foofs:1/1 {0=foofs.smithi069.nsgdqs=up:reconnect} 2024-06-11T14:05:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:21 smithi069 ceph-mon[66891]: reconnect by client.14558 172.21.15.69:0/1746076871 after 0 2024-06-11T14:05:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:21 smithi069 ceph-mon[66891]: reconnect by client.24351 172.21.15.135:0/4007316783 after 0.000999977 2024-06-11T14:05:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:21 smithi069 ceph-mon[66891]: pgmap v218: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 8.6 KiB/s wr, 7 op/s 2024-06-11T14:05:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:21 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:rejoin 2024-06-11T14:05:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:21 smithi069 ceph-mon[66891]: fsmap foofs:1/1 {0=foofs.smithi069.nsgdqs=up:rejoin} 2024-06-11T14:05:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:21 smithi069 ceph-mon[66891]: daemon mds.foofs.smithi069.nsgdqs is now active in filesystem foofs as rank 0 2024-06-11T14:05:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:21 smithi135 ceph-mon[53782]: reconnect by client.14558 172.21.15.69:0/1746076871 after 0 2024-06-11T14:05:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:21 smithi135 ceph-mon[53782]: reconnect by client.24351 172.21.15.135:0/4007316783 after 0.000999977 2024-06-11T14:05:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:21 smithi135 ceph-mon[53782]: pgmap v218: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 8.6 KiB/s wr, 7 op/s 2024-06-11T14:05:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:21 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:rejoin 2024-06-11T14:05:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:21 smithi135 ceph-mon[53782]: fsmap foofs:1/1 {0=foofs.smithi069.nsgdqs=up:rejoin} 2024-06-11T14:05:21.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:21 smithi135 ceph-mon[53782]: daemon mds.foofs.smithi069.nsgdqs is now active in filesystem foofs as rank 0 2024-06-11T14:05:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:22 smithi069 ceph-mon[66891]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-11T14:05:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:22 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:active 2024-06-11T14:05:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:22 smithi069 ceph-mon[66891]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 2024-06-11T14:05:22.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:22 smithi135 ceph-mon[53782]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-06-11T14:05:22.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:22 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278] up:active 2024-06-11T14:05:22.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:22 smithi135 ceph-mon[53782]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 2024-06-11T14:05:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:23 smithi069 ceph-mon[66891]: pgmap v219: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 204 B/s wr, 5 op/s 2024-06-11T14:05:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:23 smithi135 ceph-mon[53782]: pgmap v219: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 204 B/s wr, 5 op/s 2024-06-11T14:05:24.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:24 smithi069 ceph-mon[66891]: pgmap v220: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 7 op/s 2024-06-11T14:05:24.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:24 smithi135 ceph-mon[53782]: pgmap v220: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 7 op/s 2024-06-11T14:05:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:27 smithi069 ceph-mon[66891]: pgmap v221: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 11 op/s 2024-06-11T14:05:27.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:27 smithi135 ceph-mon[53782]: pgmap v221: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 11 op/s 2024-06-11T14:05:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:29 smithi069 ceph-mon[66891]: pgmap v222: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 11 op/s 2024-06-11T14:05:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:29 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:29 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:29 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:29 smithi135 ceph-mon[53782]: pgmap v222: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 11 op/s 2024-06-11T14:05:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:29 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:29 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:29 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:30.432 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:30 smithi135 ceph-mon[53782]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-11T14:05:30.433 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:30 smithi135 ceph-mon[53782]: Cluster is now healthy 2024-06-11T14:05:30.433 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:30 smithi135 ceph-mon[53782]: mds.? [v2:172.21.15.135:6832/235562329,v1:172.21.15.135:6833/235562329] up:boot 2024-06-11T14:05:30.433 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:30 smithi135 ceph-mon[53782]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T14:05:30.433 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:30 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T14:05:30.433 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:05:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:30 smithi069 ceph-mon[66891]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-06-11T14:05:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:30 smithi069 ceph-mon[66891]: Cluster is now healthy 2024-06-11T14:05:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:30 smithi069 ceph-mon[66891]: mds.? [v2:172.21.15.135:6832/235562329,v1:172.21.15.135:6833/235562329] up:boot 2024-06-11T14:05:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:30 smithi069 ceph-mon[66891]: fsmap foofs:1 {0=foofs.smithi069.nsgdqs=up:active} 1 up:standby 2024-06-11T14:05:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:30 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (7m) 20s ago 29m 16.8M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (5m) 20s ago 29m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (5m) 41s ago 28m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (7m) 20s ago 28m 43.6M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (25s) 20s ago 23m 13.9M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 29663d8e23b4 2024-06-11T14:05:31.080 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 starting - - - - 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (6m) 20s ago 30m 507M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (6m) 41s ago 28m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (6m) 20s ago 30m 56.1M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (5m) 41s ago 28m 43.2M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (22m) 20s ago 22m 299M - 4.0 e1d6a67b021e bba494a6f6c5 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (22m) 41s ago 22m 87.8M - 4.0 e1d6a67b021e fb0c988f9ddd 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (6m) 20s ago 28m 11.0M - 1.3.1 1dbe0e931976 fdaa371c7d5b 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (6m) 41s ago 28m 15.5M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (4m) 20s ago 27m 214M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (4m) 20s ago 27m 191M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (3m) 20s ago 26m 170M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (2m) 20s ago 26m 206M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (2m) 41s ago 25m 228M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:05:31.081 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (111s) 41s ago 25m 202M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:05:31.082 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (78s) 41s ago 25m 194M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:05:31.082 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (45s) 41s ago 24m 11.2M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:05:31.082 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (6m) 20s ago 27m 76.9M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:05:31.479 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:31 smithi069 ceph-mon[66891]: pgmap v223: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 173 B/s wr, 9 op/s 2024-06-11T14:05:31.479 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:05:31.479 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:05:31.479 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:05:31.479 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:05:31.479 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 14 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:05:31.480 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:05:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:31 smithi135 ceph-mon[53782]: pgmap v223: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 173 B/s wr, 9 op/s 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:05:31.808 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-06-11T14:05:31.809 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-06-11T14:05:31.809 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:05:31.809 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "15/23 daemons upgraded", 2024-06-11T14:05:31.809 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading mds daemons", 2024-06-11T14:05:31.809 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:05:31.809 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:05:32.196 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:05:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1810584402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:32.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:32.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:32.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:32 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/118701475' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1810584402' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:32.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:32 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/118701475' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:05:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:33 smithi069 ceph-mon[66891]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:33 smithi069 ceph-mon[66891]: pgmap v224: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-06-11T14:05:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:33 smithi135 ceph-mon[53782]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:05:33.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:33 smithi135 ceph-mon[53782]: pgmap v224: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-06-11T14:05:33.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:33.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:35 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:35.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:35 smithi135 ceph-mon[53782]: pgmap v225: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-06-11T14:05:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:35 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:35 smithi069 ceph-mon[66891]: pgmap v225: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-06-11T14:05:36.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: Detected new or changed devices on smithi135 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all mds 2024-06-11T14:05:36.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.nsgdqs"}]': finished 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi135.wtrawd"}]': finished 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all rgw 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all rbd-mirror 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all ceph-exporter 2024-06-11T14:05:37.000 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all iscsi 2024-06-11T14:05:37.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.001 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:36 smithi069 ceph-mon[66891]: pgmap v226: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 3.8 KiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: Detected new or changed devices on smithi135 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all mds 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.nsgdqs"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi069.nsgdqs"}]': finished 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi135.wtrawd"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi135.wtrawd"}]': finished 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all rgw 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all rbd-mirror 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all ceph-exporter 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.050 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:37.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all iscsi 2024-06-11T14:05:37.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:37.051 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:36 smithi135 ceph-mon[53782]: pgmap v226: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 3.8 KiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-06-11T14:05:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Upgrade: Updating nfs.foo.0.0.smithi069.txqrqg (1/2) 2024-06-11T14:05:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg 2024-06-11T14:05:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T14:05:38.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Rados config object exists: conf-nfs.foo 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg-rgw 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Bind address in nfs.foo.0.0.smithi069.txqrqg's ganesha conf is defaulting to empty 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: Deploying daemon nfs.foo.0.0.smithi069.txqrqg on smithi069 2024-06-11T14:05:38.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:38 smithi069 ceph-mon[66891]: pgmap v227: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-06-11T14:05:38.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Upgrade: Updating nfs.foo.0.0.smithi069.txqrqg (1/2) 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Rados config object exists: conf-nfs.foo 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Creating key for client.nfs.foo.0.0.smithi069.txqrqg-rgw 2024-06-11T14:05:38.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi069.txqrqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T14:05:38.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Bind address in nfs.foo.0.0.smithi069.txqrqg's ganesha conf is defaulting to empty 2024-06-11T14:05:38.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:38.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: Deploying daemon nfs.foo.0.0.smithi069.txqrqg on smithi069 2024-06-11T14:05:38.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:38 smithi135 ceph-mon[53782]: pgmap v227: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-06-11T14:05:41.484 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:41 smithi135 ceph-mon[53782]: pgmap v228: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 1.1 KiB/s rd, 7.3 KiB/s wr, 4 op/s 2024-06-11T14:05:41.484 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:41 smithi135 ceph-mon[53782]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/1746076871"}]: dispatch 2024-06-11T14:05:41.485 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:41.485 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:41 smithi069 ceph-mon[66891]: pgmap v228: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 1.1 KiB/s rd, 7.3 KiB/s wr, 4 op/s 2024-06-11T14:05:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:41 smithi069 ceph-mon[66891]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/1746076871"}]: dispatch 2024-06-11T14:05:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:42.480 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:42 smithi069 ceph-mon[66891]: Evicting (and blocklisting) client session 14558 (172.21.15.69:0/1746076871) 2024-06-11T14:05:42.480 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:42 smithi069 ceph-mon[66891]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/1746076871"}]': finished 2024-06-11T14:05:42.481 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:42 smithi069 ceph-mon[66891]: osdmap e127: 8 total, 8 up, 8 in 2024-06-11T14:05:42.486 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:42 smithi135 ceph-mon[53782]: Evicting (and blocklisting) client session 14558 (172.21.15.69:0/1746076871) 2024-06-11T14:05:42.487 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:42 smithi135 ceph-mon[53782]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.69:0/1746076871"}]': finished 2024-06-11T14:05:42.487 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:42 smithi135 ceph-mon[53782]: osdmap e127: 8 total, 8 up, 8 in 2024-06-11T14:05:43.487 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: pgmap v230: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 511 B/s wr, 1 op/s 2024-06-11T14:05:43.488 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Upgrade: Updating nfs.foo.1.0.smithi135.mndnps (2/2) 2024-06-11T14:05:43.488 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:43.488 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Creating key for client.nfs.foo.1.0.smithi135.mndnps 2024-06-11T14:05:43.488 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T14:05:43.488 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-11T14:05:43.489 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T14:05:43.489 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T14:05:43.489 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:43.489 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T14:05:43.489 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T14:05:43.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Rados config object exists: conf-nfs.foo 2024-06-11T14:05:43.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Creating key for client.nfs.foo.1.0.smithi135.mndnps-rgw 2024-06-11T14:05:43.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T14:05:43.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Bind address in nfs.foo.1.0.smithi135.mndnps's ganesha conf is defaulting to empty 2024-06-11T14:05:43.490 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:43.491 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:43 smithi135 ceph-mon[53782]: Deploying daemon nfs.foo.1.0.smithi135.mndnps on smithi135 2024-06-11T14:05:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: pgmap v230: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 511 B/s wr, 1 op/s 2024-06-11T14:05:43.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Upgrade: Updating nfs.foo.1.0.smithi135.mndnps (2/2) 2024-06-11T14:05:43.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:43.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Creating key for client.nfs.foo.1.0.smithi135.mndnps 2024-06-11T14:05:43.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-06-11T14:05:43.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-06-11T14:05:43.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-06-11T14:05:43.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-06-11T14:05:43.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:43.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-06-11T14:05:43.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-06-11T14:05:43.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Rados config object exists: conf-nfs.foo 2024-06-11T14:05:43.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Creating key for client.nfs.foo.1.0.smithi135.mndnps-rgw 2024-06-11T14:05:43.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi135.mndnps-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-06-11T14:05:43.501 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Bind address in nfs.foo.1.0.smithi135.mndnps's ganesha conf is defaulting to empty 2024-06-11T14:05:43.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:43.502 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:43 smithi069 ceph-mon[66891]: Deploying daemon nfs.foo.1.0.smithi135.mndnps on smithi135 2024-06-11T14:05:44.338 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:44 smithi135 ceph-mon[53782]: pgmap v231: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 511 B/s wr, 1 op/s 2024-06-11T14:05:44.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:44 smithi069 ceph-mon[66891]: pgmap v231: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 511 B/s wr, 1 op/s 2024-06-11T14:05:47.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:47.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:47.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:47 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:47.225 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:47 smithi135 ceph-mon[53782]: pgmap v232: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-06-11T14:05:47.406 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:47.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:47.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:47 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:47.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:47 smithi069 ceph-mon[66891]: pgmap v232: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-06-11T14:05:49.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:49 smithi135 ceph-mon[53782]: pgmap v233: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-06-11T14:05:49.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:49.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:49.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:49 smithi135 ceph-mon[53782]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.135:0/4007316783"}]: dispatch 2024-06-11T14:05:49.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:49 smithi069 ceph-mon[66891]: pgmap v233: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-06-11T14:05:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:49 smithi069 ceph-mon[66891]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.135:0/4007316783"}]: dispatch 2024-06-11T14:05:49.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:05:50.721 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:50 smithi069 ceph-mon[66891]: Evicting (and blocklisting) client session 24351 (172.21.15.135:0/4007316783) 2024-06-11T14:05:50.721 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:50 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:50.722 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:50 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:50.722 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:50 smithi069 ceph-mon[66891]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.135:0/4007316783"}]': finished 2024-06-11T14:05:50.722 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:50 smithi069 ceph-mon[66891]: osdmap e128: 8 total, 8 up, 8 in 2024-06-11T14:05:50.722 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:50 smithi069 ceph-mon[66891]: pgmap v235: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-06-11T14:05:50.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:50 smithi135 ceph-mon[53782]: Evicting (and blocklisting) client session 24351 (172.21.15.135:0/4007316783) 2024-06-11T14:05:50.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:50 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:50.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:50 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:50.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:50 smithi135 ceph-mon[53782]: from='mds.? [v2:172.21.15.69:6834/4114417278,v1:172.21.15.69:6835/4114417278]' entity='mds.foofs.smithi069.nsgdqs' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.135:0/4007316783"}]': finished 2024-06-11T14:05:50.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:50 smithi135 ceph-mon[53782]: osdmap e128: 8 total, 8 up, 8 in 2024-06-11T14:05:50.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:50 smithi135 ceph-mon[53782]: pgmap v235: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-06-11T14:05:52.177 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.177 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.177 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.177 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:51 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:52.184 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:51 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:53.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:52 smithi069 ceph-mon[66891]: pgmap v236: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-06-11T14:05:53.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:52 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:53.179 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:52 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:53.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:52 smithi135 ceph-mon[53782]: pgmap v236: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-06-11T14:05:53.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:52 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:53.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:52 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: pgmap v237: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.471 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.472 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.txqrqg"}]: dispatch 2024-06-11T14:05:55.473 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.txqrqg"}]': finished 2024-06-11T14:05:55.473 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi135.mndnps"}]: dispatch 2024-06-11T14:05:55.473 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi135.mndnps"}]': finished 2024-06-11T14:05:55.473 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.473 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:55 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: pgmap v237: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.txqrqg"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi069.txqrqg"}]': finished 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi135.mndnps"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi135.mndnps"}]': finished 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:05:55.552 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:55 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:05:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:56 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all nfs 2024-06-11T14:05:56.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:56 smithi069 ceph-mon[66891]: Upgrade: Setting container_image for all nvmeof 2024-06-11T14:05:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:56 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all nfs 2024-06-11T14:05:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:56 smithi135 ceph-mon[53782]: Upgrade: Setting container_image for all nvmeof 2024-06-11T14:05:57.325 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:57 smithi069 ceph-mon[66891]: pgmap v238: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-06-11T14:05:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:57 smithi135 ceph-mon[53782]: pgmap v238: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-06-11T14:05:58.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:58 smithi069 ceph-mon[66891]: Upgrade: Updating node-exporter.smithi069 (1/2) 2024-06-11T14:05:58.315 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:58 smithi069 ceph-mon[66891]: Deploying daemon node-exporter.smithi069 on smithi069 2024-06-11T14:05:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:58 smithi135 ceph-mon[53782]: Upgrade: Updating node-exporter.smithi069 (1/2) 2024-06-11T14:05:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:58 smithi135 ceph-mon[53782]: Deploying daemon node-exporter.smithi069 on smithi069 2024-06-11T14:05:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:05:59 smithi069 ceph-mon[66891]: pgmap v239: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-06-11T14:05:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:05:59 smithi135 ceph-mon[53782]: pgmap v239: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-06-11T14:06:00.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:00 smithi069 ceph-mon[66891]: pgmap v240: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 298 B/s rd, 397 B/s wr, 0 op/s 2024-06-11T14:06:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:00 smithi135 ceph-mon[53782]: pgmap v240: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 298 B/s rd, 397 B/s wr, 0 op/s 2024-06-11T14:06:02.169 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:01 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:02.169 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:01 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:02.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:01 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:02.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:01 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:02.556 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:06:03.198 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (7m) 12s ago 29m 16.8M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (5m) 12s ago 29m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (5m) 14s ago 29m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (7m) 12s ago 29m 42.9M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (57s) 12s ago 23m 36.8M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 29663d8e23b4 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (34s) 14s ago 23m 13.0M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 4853ad0e9df1 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (7m) 12s ago 31m 509M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (6m) 14s ago 28m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (6m) 12s ago 31m 58.4M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (6m) 14s ago 28m 46.8M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (22s) 12s ago 23m 62.0M - 5.5 2d7459ec65a0 feea82f23e04 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (17s) 14s ago 23m 17.4M - 5.5 2d7459ec65a0 9f559f8a47fc 2024-06-11T14:06:03.199 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 starting - - - - 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (7m) 14s ago 28m 15.0M - 1.3.1 1dbe0e931976 a3c8b4641c08 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (5m) 12s ago 28m 214M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (4m) 12s ago 27m 191M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (4m) 12s ago 27m 171M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (3m) 12s ago 26m 207M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (2m) 14s ago 26m 233M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (2m) 14s ago 26m 210M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (110s) 14s ago 25m 201M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (77s) 14s ago 25m 250M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:06:03.200 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (6m) 12s ago 28m 79.1M - 2.33.4 514e6a882f6e 91f16b5660d4 2024-06-11T14:06:03.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:02 smithi069 ceph-mon[66891]: pgmap v241: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-06-11T14:06:03.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:02 smithi069 ceph-mon[66891]: Upgrade: Updating node-exporter.smithi135 (2/2) 2024-06-11T14:06:03.200 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:02 smithi069 ceph-mon[66891]: Deploying daemon node-exporter.smithi135 on smithi135 2024-06-11T14:06:03.275 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:02 smithi135 ceph-mon[53782]: pgmap v241: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-06-11T14:06:03.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:02 smithi135 ceph-mon[53782]: Upgrade: Updating node-exporter.smithi135 (2/2) 2024-06-11T14:06:03.276 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:02 smithi135 ceph-mon[53782]: Deploying daemon node-exporter.smithi135 on smithi135 2024-06-11T14:06:03.597 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:06:03.598 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:03.599 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:06:03.599 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 14 2024-06-11T14:06:03.599 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:06:03.599 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "nfs", 2024-06-11T14:06:03.934 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-06-11T14:06:03.935 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:06:03.935 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "18/23 daemons upgraded", 2024-06-11T14:06:03.935 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading node-exporter daemons", 2024-06-11T14:06:03.935 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:06:03.935 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:06:03.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:03 smithi069 ceph-mon[66891]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:03.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:03 smithi069 ceph-mon[66891]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:03.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:03 smithi069 ceph-mon[66891]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:03.935 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:03 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2119262434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:04.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:03 smithi135 ceph-mon[53782]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:04.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:03 smithi135 ceph-mon[53782]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:04.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:03 smithi135 ceph-mon[53782]: from='client.34496 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:04.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:03 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2119262434' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:04.331 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:06:05.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:04 smithi135 ceph-mon[53782]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:05.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:05.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:04 smithi135 ceph-mon[53782]: pgmap v242: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:05.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:04 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/611188552' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:06:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:04 smithi069 ceph-mon[66891]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:04 smithi069 ceph-mon[66891]: pgmap v242: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:05.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:04 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/611188552' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:06:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:07 smithi069 ceph-mon[66891]: pgmap v243: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:07 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:07 smithi135 ceph-mon[53782]: pgmap v243: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:07 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:09 smithi069 ceph-mon[66891]: pgmap v244: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:09 smithi135 ceph-mon[53782]: pgmap v244: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:11.189 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.189 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.189 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:10 smithi135 ceph-mon[53782]: pgmap v245: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:11.189 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.190 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:10 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:10 smithi069 ceph-mon[66891]: pgmap v245: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:11.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:11.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:10 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:12 smithi069 ceph-mon[66891]: pgmap v246: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:12.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:12.999 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:12 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:12 smithi135 ceph-mon[53782]: pgmap v246: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:13.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:13.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:12 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:14.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:14.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:14 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:14.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:14 smithi069 ceph-mon[66891]: pgmap v247: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:14 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:15.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:14 smithi135 ceph-mon[53782]: pgmap v247: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:16.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:16.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:16.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:06:16.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:06:16.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:16.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:16 smithi069 ceph-mon[66891]: pgmap v248: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:17.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:17.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.048 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:17.049 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:16 smithi135 ceph-mon[53782]: pgmap v248: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:19 smithi069 ceph-mon[66891]: Upgrade: Updating prometheus.smithi069 2024-06-11T14:06:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:19 smithi069 ceph-mon[66891]: Deploying daemon prometheus.smithi069 on smithi069 2024-06-11T14:06:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:19 smithi069 ceph-mon[66891]: pgmap v249: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:19 smithi135 ceph-mon[53782]: Upgrade: Updating prometheus.smithi069 2024-06-11T14:06:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:19 smithi135 ceph-mon[53782]: Deploying daemon prometheus.smithi069 on smithi069 2024-06-11T14:06:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:19 smithi135 ceph-mon[53782]: pgmap v249: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:21 smithi069 ceph-mon[66891]: pgmap v250: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:21 smithi135 ceph-mon[53782]: pgmap v250: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:23 smithi069 ceph-mon[66891]: pgmap v251: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:23 smithi135 ceph-mon[53782]: pgmap v251: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:25 smithi069 ceph-mon[66891]: pgmap v252: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:25.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:25 smithi135 ceph-mon[53782]: pgmap v252: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:26 smithi135 ceph-mon[53782]: pgmap v253: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:26.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:26 smithi069 ceph-mon[66891]: pgmap v253: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:29.445 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:29 smithi069 ceph-mon[66891]: pgmap v254: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:29 smithi135 ceph-mon[53782]: pgmap v254: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:31.414 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:31 smithi069 ceph-mon[66891]: pgmap v255: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:31 smithi135 ceph-mon[53782]: pgmap v255: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:33 smithi069 ceph-mon[66891]: pgmap v256: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:33.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:33.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:33 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:33 smithi135 ceph-mon[53782]: pgmap v256: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:33 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:34.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:34.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:34 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:34.700 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (8m) 23s ago 30m 16.8M - 0.23.0 ba2b418f427c 8adee38ee045 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (6m) 23s ago 30m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (6m) 25s ago 29m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (8m) 23s ago 29m 43.3M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (89s) 23s ago 24m 37.0M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 29663d8e23b4 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (66s) 25s ago 24m 13.1M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 4853ad0e9df1 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (7m) 23s ago 31m 510M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:06:35.374 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (7m) 25s ago 29m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (7m) 23s ago 31m 60.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (6m) 25s ago 29m 47.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (54s) 23s ago 24m 62.2M - 5.5 2d7459ec65a0 feea82f23e04 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (49s) 25s ago 23m 63.4M - 5.5 2d7459ec65a0 9f559f8a47fc 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (34s) 23s ago 29m 8028k - 1.5.0 0da6a335fe13 2bb4e38d9238 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (28s) 25s ago 29m 3590k - 1.5.0 0da6a335fe13 5af2fc95861e 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (5m) 23s ago 28m 214M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (5m) 23s ago 28m 191M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (4m) 23s ago 27m 172M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (4m) 23s ago 27m 207M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (3m) 25s ago 27m 233M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (2m) 25s ago 26m 210M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (2m) 25s ago 26m 201M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:06:35.375 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (109s) 25s ago 25m 250M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:06:35.376 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 starting - - - - 2024-06-11T14:06:35.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:35 smithi069 ceph-mon[66891]: pgmap v257: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:35.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:35 smithi069 ceph-mon[66891]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:35.376 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:35 smithi069 ceph-mon[66891]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:35 smithi135 ceph-mon[53782]: pgmap v257: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:35 smithi135 ceph-mon[53782]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:35.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:35 smithi135 ceph-mon[53782]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:35.781 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:35.782 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:06:35.783 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:06:35.783 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:06:35.783 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:06:35.783 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 14 2024-06-11T14:06:35.783 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:06:35.783 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:06:36.131 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:06:36.131 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:06:36.131 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:06:36.131 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:06:36.131 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "nfs", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "18/23 daemons upgraded", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading prometheus daemons", 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:06:36.132 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:06:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:36 smithi069 ceph-mon[66891]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:36 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2761601181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:36 smithi069 ceph-mon[66891]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:36.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:36 smithi069 ceph-mon[66891]: pgmap v258: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:36.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:36 smithi135 ceph-mon[53782]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:36.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:36 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2761601181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:36.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:36 smithi135 ceph-mon[53782]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:06:36.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:36 smithi135 ceph-mon[53782]: pgmap v258: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:36.586 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:06:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:37 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3624913381' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:06:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:37.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:37 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:37 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3624913381' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:06:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:37.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:37 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:38.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:38.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:38 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:38.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:38 smithi069 ceph-mon[66891]: pgmap v259: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:38 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:39.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:38 smithi135 ceph-mon[53782]: pgmap v259: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: pgmap v260: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:41.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.500 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:41 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: pgmap v260: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:41.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:41.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:41 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:06:42.444 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:42 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T14:06:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:42 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-06-11T14:06:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:43 smithi069 ceph-mon[66891]: Upgrade: Updating alertmanager.smithi069 2024-06-11T14:06:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:43 smithi069 ceph-mon[66891]: Deploying daemon alertmanager.smithi069 on smithi069 2024-06-11T14:06:43.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:43 smithi069 ceph-mon[66891]: pgmap v261: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:43 smithi135 ceph-mon[53782]: Upgrade: Updating alertmanager.smithi069 2024-06-11T14:06:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:43 smithi135 ceph-mon[53782]: Deploying daemon alertmanager.smithi069 on smithi069 2024-06-11T14:06:43.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:43 smithi135 ceph-mon[53782]: pgmap v261: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:44.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:44 smithi069 ceph-mon[66891]: pgmap v262: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:44.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:44 smithi135 ceph-mon[53782]: pgmap v262: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:47 smithi069 ceph-mon[66891]: pgmap v263: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:47 smithi135 ceph-mon[53782]: pgmap v263: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:49.466 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:49 smithi069 ceph-mon[66891]: pgmap v264: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:49.466 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:49 smithi135 ceph-mon[53782]: pgmap v264: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:49.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:06:51.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:51 smithi069 ceph-mon[66891]: pgmap v265: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:51.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:51 smithi135 ceph-mon[53782]: pgmap v265: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:52.341 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:52 smithi069 ceph-mon[66891]: pgmap v266: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:52.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:52 smithi135 ceph-mon[53782]: pgmap v266: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:53.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:53 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:53.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:53 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:53.923 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:53 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:54.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:53 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:54.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:53 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:54.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:53 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:06:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:54 smithi069 ceph-mon[66891]: pgmap v267: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:55.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:54 smithi135 ceph-mon[53782]: pgmap v267: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:57 smithi069 ceph-mon[66891]: pgmap v268: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:57.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:57 smithi135 ceph-mon[53782]: pgmap v268: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:06:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:59 smithi069 ceph-mon[66891]: pgmap v269: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:59 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:59.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:06:59 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:59 smithi135 ceph-mon[53782]: pgmap v269: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:06:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:59 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:06:59.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:06:59 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:00.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:00 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:00.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:00 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:00.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:00 smithi069 ceph-mon[66891]: pgmap v270: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:00.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:00 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:00.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:00 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:00.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:00 smithi135 ceph-mon[53782]: pgmap v270: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:03.406 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: pgmap v271: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.408 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:03 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: pgmap v271: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:03.549 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:03 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:04 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T14:07:04.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:04 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-06-11T14:07:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:05 smithi069 ceph-mon[66891]: Upgrade: Updating grafana.smithi069 2024-06-11T14:07:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:05 smithi069 ceph-mon[66891]: Deploying daemon grafana.smithi069 on smithi069 2024-06-11T14:07:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:05 smithi069 ceph-mon[66891]: pgmap v272: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:05 smithi135 ceph-mon[53782]: Upgrade: Updating grafana.smithi069 2024-06-11T14:07:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:05 smithi135 ceph-mon[53782]: Deploying daemon grafana.smithi069 on smithi069 2024-06-11T14:07:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:05 smithi135 ceph-mon[53782]: pgmap v272: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:06.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:06 smithi069 ceph-mon[66891]: pgmap v273: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:06.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:06 smithi135 ceph-mon[53782]: pgmap v273: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:06.990 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:07:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:07 smithi069 ceph-mon[66891]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:07 smithi135 ceph-mon[53782]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (15s) 8s ago 30m 18.1M - 0.25.0 c8568f914cd2 799f87886588 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (6m) 8s ago 30m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (6m) 57s ago 30m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (8m) 8s ago 30m 43.2M - 8.3.5 dad864ee21e9 be3eb810ee0f 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (2m) 8s ago 24m 37.4M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 29663d8e23b4 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (99s) 57s ago 24m 13.1M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 4853ad0e9df1 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (8m) 8s ago 32m 512M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (7m) 57s ago 30m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (7m) 8s ago 32m 61.2M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (7m) 57s ago 29m 47.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:07:07.715 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (87s) 8s ago 24m 62.6M - 5.5 2d7459ec65a0 feea82f23e04 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (82s) 57s ago 24m 63.4M - 5.5 2d7459ec65a0 9f559f8a47fc 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (67s) 8s ago 30m 8116k - 1.5.0 0da6a335fe13 2bb4e38d9238 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (61s) 57s ago 29m 3590k - 1.5.0 0da6a335fe13 5af2fc95861e 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (6m) 8s ago 29m 214M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (5m) 8s ago 28m 192M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (5m) 8s ago 28m 172M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (4m) 8s ago 27m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (4m) 57s ago 27m 233M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (3m) 57s ago 27m 210M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (2m) 57s ago 26m 201M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (2m) 57s ago 26m 250M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:07:07.716 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (35s) 8s ago 29m 50.9M - 2.43.0 a07b618ecd1d d0c3c4eb70fe 2024-06-11T14:07:08.190 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:07:08.191 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:08.192 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:07:08.192 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 14 2024-06-11T14:07:08.192 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:07:08.192 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:07:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:08 smithi069 ceph-mon[66891]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:08 smithi069 ceph-mon[66891]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:08 smithi069 ceph-mon[66891]: pgmap v274: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:08.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:08 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/585486467' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:08 smithi135 ceph-mon[53782]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:08 smithi135 ceph-mon[53782]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:08 smithi135 ceph-mon[53782]: pgmap v274: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:08.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:08 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/585486467' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:08.589 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:07:08.589 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", 2024-06-11T14:07:08.589 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": true, 2024-06-11T14:07:08.589 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "Upgrading all daemon types on all hosts", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [ 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "mds", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "mon", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "crash", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "nfs", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "osd" 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": "18/23 daemons upgraded", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "Currently upgrading grafana daemons", 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:07:08.590 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:07:08.994 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:07:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:09 smithi135 ceph-mon[53782]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:09.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:09 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1008096543' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:07:09.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:09 smithi069 ceph-mon[66891]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:09.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:09 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1008096543' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:07:10.548 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:10 smithi135 ceph-mon[53782]: pgmap v275: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:10.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:10 smithi069 ceph-mon[66891]: pgmap v275: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:13 smithi069 ceph-mon[66891]: pgmap v276: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:13 smithi135 ceph-mon[53782]: pgmap v276: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:14.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:14 smithi135 ceph-mon[53782]: pgmap v277: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:14.567 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:14 smithi069 ceph-mon[66891]: pgmap v277: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:17.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:17.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:17.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:16 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:17.249 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:16 smithi069 ceph-mon[66891]: pgmap v278: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:17.251 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:17.251 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:17.251 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:16 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:17.251 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:16 smithi135 ceph-mon[53782]: pgmap v278: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:19.448 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:19 smithi069 ceph-mon[66891]: pgmap v279: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:19.448 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:19 smithi135 ceph-mon[53782]: pgmap v279: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:20.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:20 smithi069 ceph-mon[66891]: pgmap v280: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:20.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:20 smithi135 ceph-mon[53782]: pgmap v280: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:22.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:22 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.673 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:22 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:22 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:22 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.674 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:22 smithi069 ceph-mon[66891]: pgmap v281: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:22 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:22 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:22 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:22 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:22.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:22 smithi135 ceph-mon[53782]: pgmap v281: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:25 smithi069 ceph-mon[66891]: pgmap v282: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:25.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:25 smithi135 ceph-mon[53782]: pgmap v282: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.749 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.750 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: Upgrade: Finalizing container_image settings 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.751 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-11T14:07:26.752 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: Upgrade: Complete! 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-11T14:07:26.753 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.754 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:26 smithi069 ceph-mon[66891]: pgmap v283: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:26.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.799 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: Upgrade: Finalizing container_image settings 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-06-11T14:07:26.800 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-06-11T14:07:26.801 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: Upgrade: Complete! 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-06-11T14:07:26.802 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:26.803 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:26 smithi135 ceph-mon[53782]: pgmap v283: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:29 smithi069 ceph-mon[66891]: pgmap v284: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:29 smithi135 ceph-mon[53782]: pgmap v284: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:30 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:30.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:30 smithi069 ceph-mon[66891]: pgmap v285: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:30.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:30 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:07:30.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:30 smithi135 ceph-mon[53782]: pgmap v285: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:33 smithi069 ceph-mon[66891]: pgmap v286: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:33 smithi135 ceph-mon[53782]: pgmap v286: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:34.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:34.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:34 smithi069 ceph-mon[66891]: pgmap v287: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:34.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:34 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:34.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:34 smithi135 ceph-mon[53782]: pgmap v287: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:36.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:36 smithi135 ceph-mon[53782]: pgmap v288: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:36.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:36 smithi069 ceph-mon[66891]: pgmap v288: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:38.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:38 smithi135 ceph-mon[53782]: pgmap v289: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:38 smithi069 ceph-mon[66891]: pgmap v289: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:40.102 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph orch ps' 2024-06-11T14:07:40.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:40 smithi069 ceph-mon[66891]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:40.415 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:40 smithi069 ceph-mon[66891]: pgmap v290: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:40.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:40 smithi135 ceph-mon[53782]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:40.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:40 smithi135 ceph-mon[53782]: pgmap v290: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (50s) 20s ago 31m 18.1M - 0.25.0 c8568f914cd2 799f87886588 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (7m) 20s ago 31m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (7m) 92s ago 30m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (26s) 20s ago 31m 32.8M - 9.4.12 2bacad6d85d8 e1dae01be3f2 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (2m) 20s ago 25m 37.5M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 29663d8e23b4 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (2m) 92s ago 25m 13.1M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 4853ad0e9df1 2024-06-11T14:07:42.124 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (8m) 20s ago 32m 520M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (8m) 92s ago 30m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (8m) 20s ago 32m 61.9M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (7m) 92s ago 30m 47.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (2m) 20s ago 25m 62.7M - 5.5 2d7459ec65a0 feea82f23e04 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (116s) 92s ago 25m 63.4M - 5.5 2d7459ec65a0 9f559f8a47fc 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (101s) 20s ago 30m 7964k - 1.5.0 0da6a335fe13 2bb4e38d9238 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (95s) 92s ago 30m 3590k - 1.5.0 0da6a335fe13 5af2fc95861e 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (6m) 20s ago 29m 214M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (6m) 20s ago 29m 192M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (5m) 20s ago 28m 172M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (5m) 20s ago 28m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (4m) 92s ago 28m 233M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (4m) 92s ago 27m 210M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:07:42.125 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (3m) 92s ago 27m 201M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:07:42.126 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (2m) 92s ago 26m 250M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:07:42.126 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (70s) 20s ago 30m 57.6M - 2.43.0 a07b618ecd1d d0c3c4eb70fe 2024-06-11T14:07:42.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:42 smithi069 ceph-mon[66891]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:42.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:42 smithi069 ceph-mon[66891]: pgmap v291: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:42 smithi135 ceph-mon[53782]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:07:42.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:42 smithi135 ceph-mon[53782]: pgmap v291: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:42.762 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph versions' 2024-06-11T14:07:44.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:44 smithi135 ceph-mon[53782]: pgmap v292: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:44.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:44 smithi069 ceph-mon[66891]: pgmap v292: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:46.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:46 smithi135 ceph-mon[53782]: pgmap v293: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:46.548 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:46 smithi069 ceph-mon[66891]: pgmap v293: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:07:47.206 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 14 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:07:47.207 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:07:47.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:47 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/3221568521' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:47.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:47 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/3221568521' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:07:47.772 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-06-11T14:07:48.334 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:48 smithi069 ceph-mon[66891]: pgmap v294: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:48.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:48 smithi135 ceph-mon[53782]: pgmap v294: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:49.484 INFO:teuthology.orchestra.run.smithi069.stdout:wait for servicemap items w/ changing names to refresh 2024-06-11T14:07:49.661 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:49.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:07:50.119 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'sleep 60' 2024-06-11T14:07:50.417 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:50 smithi069 ceph-mon[66891]: pgmap v295: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:50.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:50 smithi135 ceph-mon[53782]: pgmap v295: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:52.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:52 smithi135 ceph-mon[53782]: pgmap v296: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:52.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:52 smithi069 ceph-mon[66891]: pgmap v296: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:54.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:54 smithi135 ceph-mon[53782]: pgmap v297: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:54.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:54 smithi069 ceph-mon[66891]: pgmap v297: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:56.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:56 smithi135 ceph-mon[53782]: pgmap v298: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:56.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:56 smithi069 ceph-mon[66891]: pgmap v298: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:07:58.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:07:58 smithi135 ceph-mon[53782]: pgmap v299: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:07:58.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:07:58 smithi069 ceph-mon[66891]: pgmap v299: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:00.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:00 smithi135 ceph-mon[53782]: pgmap v300: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:00.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:00 smithi069 ceph-mon[66891]: pgmap v300: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:02.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:02 smithi135 ceph-mon[53782]: pgmap v301: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:02.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:02 smithi069 ceph-mon[66891]: pgmap v301: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:04.499 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:04.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:05 smithi069 ceph-mon[66891]: pgmap v302: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:05.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:05 smithi135 ceph-mon[53782]: pgmap v302: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:07.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:07 smithi069 ceph-mon[66891]: pgmap v303: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:07.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:07 smithi135 ceph-mon[53782]: pgmap v303: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:09.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:09 smithi069 ceph-mon[66891]: pgmap v304: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:09 smithi135 ceph-mon[53782]: pgmap v304: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:11 smithi069 ceph-mon[66891]: pgmap v305: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:11.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:11 smithi135 ceph-mon[53782]: pgmap v305: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:13.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:13 smithi069 ceph-mon[66891]: pgmap v306: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:13.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:13 smithi135 ceph-mon[53782]: pgmap v306: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:15.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:15 smithi069 ceph-mon[66891]: pgmap v307: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:15.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:15 smithi135 ceph-mon[53782]: pgmap v307: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:17.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:17 smithi069 ceph-mon[66891]: pgmap v308: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:17.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:17 smithi135 ceph-mon[53782]: pgmap v308: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:19 smithi069 ceph-mon[66891]: pgmap v309: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:19.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:19 smithi135 ceph-mon[53782]: pgmap v309: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:19.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:21.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:21 smithi069 ceph-mon[66891]: pgmap v310: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:21.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:21 smithi135 ceph-mon[53782]: pgmap v310: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:23.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:23 smithi069 ceph-mon[66891]: pgmap v311: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:23.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:23 smithi135 ceph-mon[53782]: pgmap v311: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:25.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:25 smithi069 ceph-mon[66891]: pgmap v312: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:25.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:25 smithi135 ceph-mon[53782]: pgmap v312: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:08:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:26.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:26 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-06-11T14:08:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:26.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:26 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:27 smithi069 ceph-mon[66891]: pgmap v313: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:27 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:08:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:27 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:08:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:27 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:27.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:27 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:27.550 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:27 smithi135 ceph-mon[53782]: pgmap v313: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:27.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:27 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-06-11T14:08:27.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:27 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-06-11T14:08:27.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:27 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:27.551 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:27 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' 2024-06-11T14:08:29.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:29 smithi069 ceph-mon[66891]: pgmap v314: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:29.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:29 smithi135 ceph-mon[53782]: pgmap v314: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:31.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:31 smithi069 ceph-mon[66891]: pgmap v315: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:31.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:31 smithi135 ceph-mon[53782]: pgmap v315: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:33.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:33 smithi069 ceph-mon[66891]: pgmap v316: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:33.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:33 smithi135 ceph-mon[53782]: pgmap v316: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:34.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:34 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:34.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:34 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:35.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:35 smithi069 ceph-mon[66891]: pgmap v317: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:35.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:35 smithi135 ceph-mon[53782]: pgmap v317: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:36.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:36 smithi135 ceph-mon[53782]: pgmap v318: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:36.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:36 smithi069 ceph-mon[66891]: pgmap v318: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:38.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:38 smithi069 ceph-mon[66891]: pgmap v319: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:38.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:38 smithi135 ceph-mon[53782]: pgmap v319: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:40.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:40 smithi069 ceph-mon[66891]: pgmap v320: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:40.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:40 smithi135 ceph-mon[53782]: pgmap v320: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:42.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:42 smithi069 ceph-mon[66891]: pgmap v321: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:42.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:42 smithi135 ceph-mon[53782]: pgmap v321: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:44.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:44 smithi069 ceph-mon[66891]: pgmap v322: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:44.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:44 smithi135 ceph-mon[53782]: pgmap v322: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:46.748 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:46 smithi069 ceph-mon[66891]: pgmap v323: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:46.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:46 smithi135 ceph-mon[53782]: pgmap v323: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:48.550 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:48 smithi069 ceph-mon[66891]: pgmap v324: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:48.600 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:48 smithi135 ceph-mon[53782]: pgmap v324: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:49.564 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:49 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:49.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:49 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:08:50.706 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:50 smithi069 ceph-mon[66891]: pgmap v325: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:50.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:50 smithi135 ceph-mon[53782]: pgmap v325: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:52.407 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:52 smithi069 ceph-mon[66891]: pgmap v326: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:52.435 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph orch ps' 2024-06-11T14:08:52.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:52 smithi135 ceph-mon[53782]: pgmap v326: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:54.426 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:54 smithi069 ceph-mon[66891]: pgmap v327: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:54.426 INFO:teuthology.orchestra.run.smithi069.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:alertmanager.smithi069 smithi069 *:9093,9094 running (2m) 93s ago 32m 18.1M - 0.25.0 c8568f914cd2 799f87886588 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi069 smithi069 running (8m) 93s ago 32m 6899k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0dcc488b2927 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:crash.smithi135 smithi135 running (8m) 2m ago 31m 6895k - 19.0.0-2976-g3f51ea39 2d7459ec65a0 3b4fe53e360e 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:grafana.smithi069 smithi069 *:3000 running (99s) 93s ago 32m 32.8M - 9.4.12 2bacad6d85d8 e1dae01be3f2 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi069.nsgdqs smithi069 running (3m) 93s ago 26m 37.5M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 29663d8e23b4 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:mds.foofs.smithi135.wtrawd smithi135 running (3m) 2m ago 26m 13.1M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 4853ad0e9df1 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi069.iruzyw smithi069 *:8443,9283,8765 running (10m) 93s ago 33m 520M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 6306593cf6f0 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:mgr.smithi135.xcould smithi135 *:8443,9283,8765 running (9m) 2m ago 31m 441M - 19.0.0-2976-g3f51ea39 2d7459ec65a0 0c9d9a81c3aa 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi069 smithi069 running (9m) 93s ago 33m 61.9M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 f050c71c1242 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:mon.smithi135 smithi135 running (9m) 2m ago 31m 47.0M 2048M 19.0.0-2976-g3f51ea39 2d7459ec65a0 a50d298981f4 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.0.0.smithi069.txqrqg smithi069 *:2049 running (3m) 93s ago 26m 62.7M - 5.5 2d7459ec65a0 feea82f23e04 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:nfs.foo.1.0.smithi135.mndnps smithi135 *:2049 running (3m) 2m ago 26m 63.4M - 5.5 2d7459ec65a0 9f559f8a47fc 2024-06-11T14:08:54.427 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi069 smithi069 *:9100 running (2m) 93s ago 32m 7964k - 1.5.0 0da6a335fe13 2bb4e38d9238 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:node-exporter.smithi135 smithi135 *:9100 running (2m) 2m ago 31m 3590k - 1.5.0 0da6a335fe13 5af2fc95861e 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.0 smithi069 running (8m) 93s ago 31m 214M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 41460a588aa7 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.1 smithi069 running (7m) 93s ago 30m 192M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 1e9b021f1427 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.2 smithi069 running (6m) 93s ago 30m 172M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 d6fd364ee54d 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.3 smithi069 running (6m) 93s ago 29m 208M 1936M 19.0.0-2976-g3f51ea39 2d7459ec65a0 e8e6e999ac73 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.4 smithi135 running (5m) 2m ago 29m 233M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 2ab311934bba 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.5 smithi135 running (5m) 2m ago 28m 210M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 99a4b288fdf1 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.6 smithi135 running (4m) 2m ago 28m 201M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 016c00269b62 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:osd.7 smithi135 running (4m) 2m ago 28m 250M 2704M 19.0.0-2976-g3f51ea39 2d7459ec65a0 02983e37cce5 2024-06-11T14:08:54.428 INFO:teuthology.orchestra.run.smithi069.stdout:prometheus.smithi069 smithi069 *:9095 running (2m) 93s ago 31m 57.6M - 2.43.0 a07b618ecd1d d0c3c4eb70fe 2024-06-11T14:08:54.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:54 smithi135 ceph-mon[53782]: pgmap v327: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:55.050 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph orch upgrade status' 2024-06-11T14:08:55.604 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:55 smithi069 ceph-mon[66891]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:08:55.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:55 smithi135 ceph-mon[53782]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:08:56.628 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:56 smithi069 ceph-mon[66891]: pgmap v328: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:56.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:56 smithi135 ceph-mon[53782]: pgmap v328: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "target_image": null, 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "in_progress": false, 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "which": "", 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "services_complete": [], 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "progress": null, 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "message": "", 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout: "is_paused": false 2024-06-11T14:08:57.181 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:08:57.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:57 smithi069 ceph-mon[66891]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:08:57.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:57 smithi135 ceph-mon[53782]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:08:57.814 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph health detail' 2024-06-11T14:08:58.452 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:58 smithi069 ceph-mon[66891]: pgmap v329: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:58.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:58 smithi135 ceph-mon[53782]: pgmap v329: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:08:59.833 INFO:teuthology.orchestra.run.smithi069.stdout:HEALTH_OK 2024-06-11T14:09:00.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:08:59 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/2640530659' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:09:00.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:08:59 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/2640530659' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-06-11T14:09:00.437 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph versions' 2024-06-11T14:09:00.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:00 smithi069 ceph-mon[66891]: pgmap v330: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:01.297 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:00 smithi135 ceph-mon[53782]: pgmap v330: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:02.580 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:02 smithi069 ceph-mon[66891]: pgmap v331: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:02.580 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-06-11T14:09:02.580 INFO:teuthology.orchestra.run.smithi069.stdout: "mon": { 2024-06-11T14:09:02.580 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:09:02.580 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:09:02.580 INFO:teuthology.orchestra.run.smithi069.stdout: "mgr": { 2024-06-11T14:09:02.580 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: "osd": { 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 8 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: "mds": { 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 2 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: "overall": { 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)": 14 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:09:02.581 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-06-11T14:09:02.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:02 smithi135 ceph-mon[53782]: pgmap v331: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:03.152 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-06-11T14:09:03.455 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:03 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/1313130511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:09:03.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:03 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/1313130511' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:09:04.460 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:04 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:09:04.461 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:04 smithi069 ceph-mon[66891]: pgmap v332: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:04.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:04 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:09:04.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:04 smithi135 ceph-mon[53782]: pgmap v332: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:05.215 INFO:teuthology.orchestra.run.smithi069.stdout:true 2024-06-11T14:09:05.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:05 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/4061928366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:09:05.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:05 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/4061928366' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:09:05.867 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -e sha1=3f51ea3944d39ca13aa40153c9a4ee298306679d -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-06-11T14:09:06.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:06 smithi069 ceph-mon[66891]: pgmap v333: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:06.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:06 smithi135 ceph-mon[53782]: pgmap v333: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:08.067 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph version 19.0.0-2976-g3f51ea39 (3f51ea3944d39ca13aa40153c9a4ee298306679d) squid (rc)" 2024-06-11T14:09:08.356 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:08 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/80316460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:09:08.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:08 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/80316460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-06-11T14:09:08.671 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-06-11T14:09:08.672 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-06-11T14:09:08.673 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- ceph orch ls -f json 2024-06-11T14:09:09.248 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:09 smithi069 ceph-mon[66891]: pgmap v334: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:09.547 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:09 smithi135 ceph-mon[53782]: pgmap v334: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:10.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:10 smithi069 ceph-mon[66891]: pgmap v335: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:10.646 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-06-11T14:09:10.646 INFO:teuthology.orchestra.run.smithi069.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-06-11T13:35:47.061980Z", "last_refresh": "2024-06-11T14:07:21.416738Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-06-11T13:35:41.037050Z", "last_refresh": "2024-06-11T14:06:09.881742Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-06-11T13:35:43.843322Z", "last_refresh": "2024-06-11T14:07:21.416883Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-06-11T13:42:08.936725Z", "last_refresh": "2024-06-11T14:06:09.882355Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-06-11T13:35:38.798707Z", "last_refresh": "2024-06-11T14:06:09.881834Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi069:172.21.15.69=smithi069", "smithi135:172.21.15.135=smithi135"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-06-11T13:36:28.693457Z", "last_refresh": "2024-06-11T14:06:09.881909Z", "running": 2, "size": 2}}, {"events": ["2024-06-11T14:08:26.496235Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-06-11T13:42:29.611025Z", "last_refresh": "2024-06-11T14:06:09.882423Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-06-11T13:35:45.126603Z", "last_refresh": "2024-06-11T14:06:09.881997Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310", "last_refresh": "2024-06-11T14:06:09.882069Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-06-11T13:35:42.191099Z", "last_refresh": "2024-06-11T14:07:21.417072Z", "ports": [9095], "running": 1, "size": 1}}] 2024-06-11T14:09:10.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:10 smithi135 ceph-mon[53782]: pgmap v335: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:11.270 INFO:tasks.cephadm:nfs.foo has 2/2 2024-06-11T14:09:11.271 DEBUG:teuthology.parallel:result is None 2024-06-11T14:09:11.271 INFO:teuthology.run_tasks:Running task vip.exec... 2024-06-11T14:09:11.281 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T14:09:11.281 DEBUG:teuthology.orchestra.run.smithi069:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-06-11T14:09:11.306 INFO:teuthology.orchestra.run.smithi069.stderr:+ umount /mnt/foo 2024-06-11T14:09:11.318 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-11T14:09:11.326 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T14:09:11.326 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-06-11T14:09:11.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:11 smithi069 ceph-mon[66891]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T14:09:11.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:11 smithi135 ceph-mon[53782]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-06-11T14:09:12.653 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:12 smithi069 ceph-mon[66891]: pgmap v336: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:12.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:12 smithi135 ceph-mon[53782]: pgmap v336: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:13.388 INFO:teuthology.orchestra.run.smithi069.stdout: "foo" 2024-06-11T14:09:13.959 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-06-11T14:09:14.605 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:14 smithi069 ceph-mon[66891]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:09:14.605 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:14 smithi069 ceph-mon[66891]: pgmap v337: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:14.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:14 smithi135 ceph-mon[53782]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:09:14.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:14 smithi135 ceph-mon[53782]: pgmap v337: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:15.941 INFO:teuthology.orchestra.run.smithi069.stdout:[ 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: { 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "access_type": "RW", 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "clients": [], 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "cluster_id": "foo", 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "export_id": 1, 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "fsal": { 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "fs_name": "foofs", 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "name": "CEPH", 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "user_id": "nfs.foo.1" 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: }, 2024-06-11T14:09:15.942 INFO:teuthology.orchestra.run.smithi069.stdout: "path": "/", 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: "protocols": [ 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: 4 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: ], 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: "pseudo": "/fake", 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: "security_label": true, 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: "squash": "none", 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: "transports": [ 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: "TCP" 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout: } 2024-06-11T14:09:15.943 INFO:teuthology.orchestra.run.smithi069.stdout:] 2024-06-11T14:09:16.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:16 smithi069 ceph-mon[66891]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:09:16.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:16 smithi069 ceph-mon[66891]: pgmap v338: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:16.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:16 smithi069 ceph-mon[66891]: mgrmap e41: smithi069.iruzyw(active, since 9m), standbys: smithi135.xcould 2024-06-11T14:09:16.623 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-06-11T14:09:16.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:16 smithi135 ceph-mon[53782]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-06-11T14:09:16.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:16 smithi135 ceph-mon[53782]: pgmap v338: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-06-11T14:09:16.798 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:16 smithi135 ceph-mon[53782]: mgrmap e41: smithi069.iruzyw(active, since 9m), standbys: smithi135.xcould 2024-06-11T14:09:18.451 INFO:teuthology.orchestra.run.smithi069.stdout:foo rec-0000000000000004:nfs.foo.1 2024-06-11T14:09:18.451 INFO:teuthology.orchestra.run.smithi069.stdout:foo grace 2024-06-11T14:09:18.451 INFO:teuthology.orchestra.run.smithi069.stdout:foo rec-0000000000000004:nfs.foo.0 2024-06-11T14:09:18.451 INFO:teuthology.orchestra.run.smithi069.stdout:foo conf-nfs.foo 2024-06-11T14:09:18.451 INFO:teuthology.orchestra.run.smithi069.stdout:foo export-1 2024-06-11T14:09:18.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:18 smithi069 ceph-mon[66891]: pgmap v339: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:18.599 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:18 smithi135 ceph-mon[53782]: pgmap v339: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-06-11T14:09:19.079 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-06-11T14:09:19.089 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T14:09:19.089 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 -- bash -c 'set -ex 2024-06-11T14:09:19.089 DEBUG:teuthology.orchestra.run.smithi069:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-06-11T14:09:19.089 DEBUG:teuthology.orchestra.run.smithi069:> ' 2024-06-11T14:09:19.616 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:19 smithi069 ceph-mon[66891]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:09:19.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:19 smithi135 ceph-mon[53782]: from='mgr.34104 172.21.15.69:0/1816067926' entity='mgr.smithi069.iruzyw' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-06-11T14:09:20.599 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:20 smithi069 ceph-mon[66891]: pgmap v340: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:09:20.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:20 smithi135 ceph-mon[53782]: pgmap v340: 97 pgs: 97 active+clean; 228 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-06-11T14:09:20.931 INFO:teuthology.orchestra.run.smithi069.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-06-11T14:09:21.177 INFO:teuthology.orchestra.run.smithi069.stderr:+ [[ 6 -gt 2 ]] 2024-06-11T14:09:21.477 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:21 smithi069 ceph-mon[66891]: from='client.? 172.21.15.69:0/4164306779' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-06-11T14:09:21.751 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-06-11T14:09:21.761 INFO:tasks.cephadm:Teardown begin 2024-06-11T14:09:21.762 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-11T14:09:21.789 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-11T14:09:21.797 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:21 smithi135 ceph-mon[53782]: from='client.? 172.21.15.69:0/4164306779' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-06-11T14:09:21.823 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-06-11T14:09:21.823 DEBUG:teuthology.orchestra.run.smithi069:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-11T14:09:21.845 DEBUG:teuthology.orchestra.run.smithi135:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-06-11T14:09:21.881 INFO:tasks.cephadm:Stopping all daemons... 2024-06-11T14:09:21.881 INFO:tasks.cephadm.mon.smithi069:Stopping mon.smithi069... 2024-06-11T14:09:21.881 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069 2024-06-11T14:09:21.998 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:21 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:22 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[66887]: 2024-06-11T14:09:22.198+0000 7fbad8461640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:22 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069[66887]: 2024-06-11T14:09:22.198+0000 7fbad8461640 -1 mon.smithi069@0(leader) e3 *** Got Signal Terminated *** 2024-06-11T14:09:22.498 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:22 smithi069 podman[112787]: 2024-06-11 14:09:22.290825665 +0000 UTC m=+0.211389987 container died f050c71c12424936a450017e1d40523059426b857852a6916b764e6702cfe54f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:09:22.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:22 smithi069 podman[112787]: 2024-06-11 14:09:22.673911915 +0000 UTC m=+0.594476230 container cleanup f050c71c12424936a450017e1d40523059426b857852a6916b764e6702cfe54f (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-06-11T14:09:22.941 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Jun 11 14:09:22 smithi069 bash[112787]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi069 2024-06-11T14:09:23.090 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi069.service' 2024-06-11T14:09:23.137 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:09:23.137 INFO:tasks.cephadm.mon.smithi069:Stopped mon.smithi069 2024-06-11T14:09:23.137 INFO:tasks.cephadm.mon.smithi135:Stopping mon.smithi135... 2024-06-11T14:09:23.137 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi135 2024-06-11T14:09:23.461 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:23 smithi135 systemd[1]: Stopping Ceph mon.smithi135 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:23.761 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135[53778]: 2024-06-11T14:09:23.460+0000 7f4bf7b0a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi135 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:23.762 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135[53778]: 2024-06-11T14:09:23.460+0000 7f4bf7b0a640 -1 mon.smithi135@1(peon) e3 *** Got Signal Terminated *** 2024-06-11T14:09:23.762 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:23 smithi135 podman[78368]: 2024-06-11 14:09:23.471633397 +0000 UTC m=+0.137570437 container died a50d298981f4958faefeb464b74ca7a52ae2dbfb65e967c5c2594cb8603704c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True) 2024-06-11T14:09:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:23 smithi135 podman[78368]: 2024-06-11 14:09:23.929010312 +0000 UTC m=+0.594947344 container cleanup a50d298981f4958faefeb464b74ca7a52ae2dbfb65e967c5c2594cb8603704c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-06-11T14:09:24.047 INFO:journalctl@ceph.mon.smithi135.smithi135.stdout:Jun 11 14:09:23 smithi135 bash[78368]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-mon-smithi135 2024-06-11T14:09:24.367 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@mon.smithi135.service' 2024-06-11T14:09:24.416 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:09:24.416 INFO:tasks.cephadm.mon.smithi135:Stopped mon.smithi135 2024-06-11T14:09:24.416 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-06-11T14:09:24.416 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.0 2024-06-11T14:09:24.736 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:24 smithi069 systemd[1]: Stopping Ceph osd.0 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:24.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:24 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:09:24.736+0000 7f4b053d7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:24.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:24 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:09:24.736+0000 7f4b053d7640 -1 osd.0 128 *** Got signal Terminated *** 2024-06-11T14:09:24.998 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:24 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0[72907]: 2024-06-11T14:09:24.736+0000 7f4b053d7640 -1 osd.0 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:09:30.579 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:30 smithi069 podman[112951]: 2024-06-11 14:09:30.280831379 +0000 UTC m=+5.670731523 container died 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, ceph=True) 2024-06-11T14:09:30.920 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:30 smithi069 podman[112951]: 2024-06-11 14:09:30.68682786 +0000 UTC m=+6.076728001 container cleanup 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, org.label-schema.build-date=20240528, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:09:30.921 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:30 smithi069 bash[112951]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0 2024-06-11T14:09:31.248 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:30 smithi069 podman[112967]: 2024-06-11 14:09:30.921715672 +0000 UTC m=+0.637436760 container remove 41460a588aa77e718463d1ace169d97fde790af7ea2dc6e55d36b05e6aff3005 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.35.3) 2024-06-11T14:09:31.566 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:31 smithi069 podman[113056]: 2024-06-11 14:09:31.164334977 +0000 UTC m=+0.017878622 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:09:31.566 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:31 smithi069 podman[113056]: 2024-06-11 14:09:31.281219964 +0000 UTC m=+0.134763630 container create 128da75e872023b4cac7c00f493f42e6cc7f977c5d1b745511652d08aca1a5f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, ceph=True, RELEASE=squid-3f51ea3) 2024-06-11T14:09:31.566 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:31 smithi069 podman[113056]: 2024-06-11 14:09:31.567339199 +0000 UTC m=+0.420882826 container init 128da75e872023b4cac7c00f493f42e6cc7f977c5d1b745511652d08aca1a5f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True) 2024-06-11T14:09:31.828 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:31 smithi069 podman[113056]: 2024-06-11 14:09:31.570780145 +0000 UTC m=+0.424323769 container start 128da75e872023b4cac7c00f493f42e6cc7f977c5d1b745511652d08aca1a5f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-06-11T14:09:31.828 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:31 smithi069 podman[113056]: 2024-06-11 14:09:31.700810583 +0000 UTC m=+0.554354211 container attach 128da75e872023b4cac7c00f493f42e6cc7f977c5d1b745511652d08aca1a5f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:09:32.187 INFO:journalctl@ceph.osd.0.smithi069.stdout:Jun 11 14:09:31 smithi069 podman[113056]: 2024-06-11 14:09:31.829519197 +0000 UTC m=+0.683062836 container died 128da75e872023b4cac7c00f493f42e6cc7f977c5d1b745511652d08aca1a5f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0) 2024-06-11T14:09:32.510 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.0.service' 2024-06-11T14:09:32.551 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:09:32.551 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-06-11T14:09:32.551 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-06-11T14:09:32.551 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.1 2024-06-11T14:09:32.748 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:32 smithi069 systemd[1]: Stopping Ceph osd.1 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:33.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:32 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:09:32.899+0000 7f2d922cb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:33.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:32 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:09:32.899+0000 7f2d922cb640 -1 osd.1 128 *** Got signal Terminated *** 2024-06-11T14:09:33.248 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:32 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1[75519]: 2024-06-11T14:09:32.899+0000 7f2d922cb640 -1 osd.1 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:09:38.748 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:38 smithi069 podman[113159]: 2024-06-11 14:09:38.484537103 +0000 UTC m=+5.700877284 container died 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:09:39.192 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:38 smithi069 podman[113159]: 2024-06-11 14:09:38.874253462 +0000 UTC m=+6.090593635 container cleanup 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.build-date=20240528) 2024-06-11T14:09:39.192 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:38 smithi069 bash[113159]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1 2024-06-11T14:09:39.498 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:39 smithi069 podman[113185]: 2024-06-11 14:09:39.19329682 +0000 UTC m=+0.703681745 container remove 1e9b021f1427e6a3b49cd4154b388011faf7fc70b07ff53e360ab75582f8a3d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux ) 2024-06-11T14:09:39.837 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:39 smithi069 podman[113273]: 2024-06-11 14:09:39.454979865 +0000 UTC m=+0.017195993 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:09:39.837 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:39 smithi069 podman[113273]: 2024-06-11 14:09:39.576627403 +0000 UTC m=+0.138843521 container create 6fbce7fb0ed6cff0bdcfdc6f439343c95e001ebdba94952f1edcef017e8d50c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_CLEAN=True) 2024-06-11T14:09:39.837 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:39 smithi069 podman[113273]: 2024-06-11 14:09:39.838212995 +0000 UTC m=+0.400429130 container init 6fbce7fb0ed6cff0bdcfdc6f439343c95e001ebdba94952f1edcef017e8d50c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, RELEASE=squid-3f51ea3) 2024-06-11T14:09:40.092 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:39 smithi069 podman[113273]: 2024-06-11 14:09:39.841732938 +0000 UTC m=+0.403949071 container start 6fbce7fb0ed6cff0bdcfdc6f439343c95e001ebdba94952f1edcef017e8d50c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, io.buildah.version=1.35.3, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-06-11T14:09:40.092 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:39 smithi069 podman[113273]: 2024-06-11 14:09:39.921039894 +0000 UTC m=+0.483256028 container attach 6fbce7fb0ed6cff0bdcfdc6f439343c95e001ebdba94952f1edcef017e8d50c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240528, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-11T14:09:40.449 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:40 smithi069 podman[113273]: 2024-06-11 14:09:40.093299544 +0000 UTC m=+0.655515672 container died 6fbce7fb0ed6cff0bdcfdc6f439343c95e001ebdba94952f1edcef017e8d50c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, org.label-schema.build-date=20240528) 2024-06-11T14:09:40.701 INFO:journalctl@ceph.osd.1.smithi069.stdout:Jun 11 14:09:40 smithi069 podman[113296]: 2024-06-11 14:09:40.699850705 +0000 UTC m=+0.597363640 container remove 6fbce7fb0ed6cff0bdcfdc6f439343c95e001ebdba94952f1edcef017e8d50c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, ceph=True, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:09:40.724 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.1.service' 2024-06-11T14:09:40.765 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:09:40.765 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-06-11T14:09:40.765 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-06-11T14:09:40.765 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.2 2024-06-11T14:09:40.998 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:40 smithi069 systemd[1]: Stopping Ceph osd.2 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:41.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:41 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:09:41.119+0000 7f3530ab2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:41.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:41 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:09:41.119+0000 7f3530ab2640 -1 osd.2 128 *** Got signal Terminated *** 2024-06-11T14:09:41.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:41 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2[78125]: 2024-06-11T14:09:41.119+0000 7f3530ab2640 -1 osd.2 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:09:46.886 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:46 smithi069 podman[113375]: 2024-06-11 14:09:46.579325197 +0000 UTC m=+5.587548356 container died d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-06-11T14:09:47.195 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:46 smithi069 podman[113375]: 2024-06-11 14:09:46.961207322 +0000 UTC m=+5.969430469 container cleanup d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-06-11T14:09:47.195 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:46 smithi069 bash[113375]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2 2024-06-11T14:09:47.498 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:47 smithi069 podman[113425]: 2024-06-11 14:09:47.195960639 +0000 UTC m=+0.610966975 container remove d6fd364ee54d7cf90b2220ee6853d113f26247920387eee80bfb391d3e34c3d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-06-11T14:09:47.848 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:47 smithi069 podman[113512]: 2024-06-11 14:09:47.530363794 +0000 UTC m=+0.111030450 container create a6e243e55db470de58d7dd4be6665e76788853bb7590fcf7639428ab3f32134b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux ) 2024-06-11T14:09:47.849 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:47 smithi069 podman[113512]: 2024-06-11 14:09:47.436394882 +0000 UTC m=+0.017061548 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:09:47.849 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:47 smithi069 podman[113512]: 2024-06-11 14:09:47.766235737 +0000 UTC m=+0.346902391 container init a6e243e55db470de58d7dd4be6665e76788853bb7590fcf7639428ab3f32134b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-06-11T14:09:47.849 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:47 smithi069 podman[113512]: 2024-06-11 14:09:47.769999589 +0000 UTC m=+0.350666258 container start a6e243e55db470de58d7dd4be6665e76788853bb7590fcf7639428ab3f32134b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, ceph=True) 2024-06-11T14:09:47.849 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:47 smithi069 podman[113512]: 2024-06-11 14:09:47.849301551 +0000 UTC m=+0.429968239 container attach a6e243e55db470de58d7dd4be6665e76788853bb7590fcf7639428ab3f32134b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3) 2024-06-11T14:09:48.248 INFO:journalctl@ceph.osd.2.smithi069.stdout:Jun 11 14:09:48 smithi069 podman[113512]: 2024-06-11 14:09:48.022043451 +0000 UTC m=+0.602710113 container died a6e243e55db470de58d7dd4be6665e76788853bb7590fcf7639428ab3f32134b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528) 2024-06-11T14:09:48.646 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.2.service' 2024-06-11T14:09:48.685 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:09:48.685 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-06-11T14:09:48.685 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-06-11T14:09:48.685 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.3 2024-06-11T14:09:48.998 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:48 smithi069 systemd[1]: Stopping Ceph osd.3 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:49.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:49 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:09:49.032+0000 7fb6b3ac2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:49.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:49 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:09:49.032+0000 7fb6b3ac2640 -1 osd.3 128 *** Got signal Terminated *** 2024-06-11T14:09:49.498 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:49 smithi069 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3[80755]: 2024-06-11T14:09:49.032+0000 7fb6b3ac2640 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:09:51.499 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:51 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:51.230+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:51.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:51 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:51.499+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:52.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:52 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:52.269+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:52.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:52 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:52.515+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:53.543 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:53 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:53.294+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:53.544 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:53 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:53.246+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:53.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:53 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:53.543+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:54.262 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:54 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:54.219+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:54.263 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:53 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:53.998+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:09:54.525 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:54 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:54.262+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:54.748 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:54 smithi069 podman[113614]: 2024-06-11 14:09:54.497252341 +0000 UTC m=+5.591076160 container died e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS) 2024-06-11T14:09:54.797 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:54 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:54.525+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:55.123 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:54 smithi069 podman[113614]: 2024-06-11 14:09:54.847306891 +0000 UTC m=+5.941130696 container cleanup e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True) 2024-06-11T14:09:55.123 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:54 smithi069 bash[113614]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3 2024-06-11T14:09:55.293 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:54 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:54.981+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:09:55.294 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:55 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:55.216+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:55.469 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:55 smithi069 podman[113644]: 2024-06-11 14:09:55.123595305 +0000 UTC m=+0.626652422 container remove e8e6e999ac734574668e9c4ff423f7683d493dfa62618ea49e661a3f0bd981dd (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:09:55.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:55 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:55.293+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:55.748 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:55 smithi069 podman[113729]: 2024-06-11 14:09:55.373175504 +0000 UTC m=+0.018304219 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:09:55.748 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:55 smithi069 podman[113729]: 2024-06-11 14:09:55.499693104 +0000 UTC m=+0.144821859 container create 018d8705ce428f61d57a54c9ad2260c4c67edae1b64d124b8f3886be1425c51d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-06-11T14:09:55.966 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:55 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:55.566+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:56.017 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:55 smithi069 podman[113729]: 2024-06-11 14:09:55.760657972 +0000 UTC m=+0.405786685 container init 018d8705ce428f61d57a54c9ad2260c4c67edae1b64d124b8f3886be1425c51d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:09:56.017 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:55 smithi069 podman[113729]: 2024-06-11 14:09:55.763862617 +0000 UTC m=+0.408991340 container start 018d8705ce428f61d57a54c9ad2260c4c67edae1b64d124b8f3886be1425c51d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-06-11T14:09:56.017 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:55 smithi069 podman[113729]: 2024-06-11 14:09:55.852430833 +0000 UTC m=+0.497559560 container attach 018d8705ce428f61d57a54c9ad2260c4c67edae1b64d124b8f3886be1425c51d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:09:56.227 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:55 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:55.967+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:09:56.372 INFO:journalctl@ceph.osd.3.smithi069.stdout:Jun 11 14:09:56 smithi069 podman[113729]: 2024-06-11 14:09:56.018505635 +0000 UTC m=+0.663634365 container died 018d8705ce428f61d57a54c9ad2260c4c67edae1b64d124b8f3886be1425c51d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-06-11T14:09:56.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:56 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:56.259+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:56.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:56 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:56.227+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:56.745 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.3.service' 2024-06-11T14:09:56.785 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:09:56.785 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-06-11T14:09:56.785 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-06-11T14:09:56.786 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.4 2024-06-11T14:09:56.833 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:56 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:56.569+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:57.230 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:56 smithi135 systemd[1]: Stopping Ceph osd.4 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:09:57.231 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:56 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:56.966+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:09:57.231 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:57 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:57.082+0000 7f4b06de5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:09:57.231 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:57 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:57.082+0000 7f4b06de5640 -1 osd.4 128 *** Got signal Terminated *** 2024-06-11T14:09:57.231 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:57 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:57.082+0000 7f4b06de5640 -1 osd.4 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:09:57.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:57 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:57.231+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:57.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:57 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:57.235+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:58.014 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:57 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:57.582+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:58.297 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:58 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:58.190+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:58.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:58 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:58.198+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:58.298 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:58 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:58.015+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:09:58.970 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:58 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:58.562+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:09:59.297 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:58 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:58.970+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:09:59.298 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:09:59 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:09:59.214+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:09:59.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:59 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:59.203+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:09:59.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:09:59 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:09:59.203+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:09:59.947 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:09:59 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:09:59.605+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:00.239 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:09:59 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:09:59.948+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:10:00.239 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:00 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:00.189+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:00.239 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:00 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:00.189+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:00.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:00 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:00.239+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:00.953 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:00 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:00.593+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:01.206 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:00 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:10:00.953+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:10:01.206 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:01.176+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:01.206 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:01.176+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:01.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:01.206+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:01.968 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:01.638+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:02.297 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:02.157+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:02.298 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:01 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4[58593]: 2024-06-11T14:10:01.969+0000 7f4aff3f3640 -1 osd.4 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:28.745621+0000 front 2024-06-11T14:09:28.745651+0000 (oldest deadline 2024-06-11T14:09:53.445516+0000) 2024-06-11T14:10:02.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:02.138+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:02.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:02.138+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:02.860 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:02 smithi135 podman[78566]: 2024-06-11 14:10:02.60656991 +0000 UTC m=+5.621959085 container died 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-3f51ea3) 2024-06-11T14:10:02.860 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:02.611+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:02.860 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:02 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:02.611+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:03.151 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78566]: 2024-06-11 14:10:03.017421319 +0000 UTC m=+6.032810493 container cleanup 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, ceph=True, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:10:03.151 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 bash[78566]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4 2024-06-11T14:10:03.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:03 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:03.204+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:03.548 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78582]: 2024-06-11 14:10:03.261327829 +0000 UTC m=+0.650188149 container remove 2ab311934bba04555efb289a5ca221f44151f1075c6384b794cb9e7fadded783 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:10:03.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:03 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:03.182+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:03.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:03 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:03.182+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:03.864 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:03 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:03.658+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:03.865 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:03 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:03.658+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:03.865 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78672]: 2024-06-11 14:10:03.497651788 +0000 UTC m=+0.017484860 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:10:03.865 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78672]: 2024-06-11 14:10:03.612541697 +0000 UTC m=+0.132374802 container create 76d91709524edc17af32c7bb4c1173b19f6ac26b3411a9c4bf812b98723f120b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-3f51ea3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528) 2024-06-11T14:10:03.865 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78672]: 2024-06-11 14:10:03.865790224 +0000 UTC m=+0.385623297 container init 76d91709524edc17af32c7bb4c1173b19f6ac26b3411a9c4bf812b98723f120b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:10:04.124 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78672]: 2024-06-11 14:10:03.86925807 +0000 UTC m=+0.389091131 container start 76d91709524edc17af32c7bb4c1173b19f6ac26b3411a9c4bf812b98723f120b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True) 2024-06-11T14:10:04.124 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:03 smithi135 podman[78672]: 2024-06-11 14:10:03.974360712 +0000 UTC m=+0.494193770 container attach 76d91709524edc17af32c7bb4c1173b19f6ac26b3411a9c4bf812b98723f120b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:10:04.486 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:04 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:04.202+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:04.486 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:04 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:04.202+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:04.486 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:04 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:04.180+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:04.487 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:04 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:04.180+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:04.487 INFO:journalctl@ceph.osd.4.smithi135.stdout:Jun 11 14:10:04 smithi135 podman[78672]: 2024-06-11 14:10:04.125162661 +0000 UTC m=+0.644995724 container died 76d91709524edc17af32c7bb4c1173b19f6ac26b3411a9c4bf812b98723f120b (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:10:04.787 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:04 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:04.678+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:04.787 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:04 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:04.678+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:04.808 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.4.service' 2024-06-11T14:10:04.847 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:10:04.847 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-06-11T14:10:04.848 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-06-11T14:10:04.848 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.5 2024-06-11T14:10:05.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:04 smithi135 systemd[1]: Stopping Ceph osd.5 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:10:05.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:05.208+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:05.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:05.208+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:05.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:05.231+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:05.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:05.231+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:05.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:05.183+0000 7f21c8cf3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:10:05.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:05.183+0000 7f21c8cf3640 -1 osd.5 128 *** Got signal Terminated *** 2024-06-11T14:10:05.548 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:05.183+0000 7f21c8cf3640 -1 osd.5 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:10:06.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:05.645+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:06.048 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:05 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:05.645+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:06.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:06 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:06.235+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:06.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:06 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:06.235+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:06.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:06 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:06.238+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:06.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:06 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:06.238+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:07.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:06 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:06.636+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:07.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:06 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:06.636+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:07.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:07.214+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:07.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:07.214+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:07.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:07.243+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:07.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:07.243+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:08.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:07.635+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:08.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:07.636+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:08.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:07 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:07.636+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:42.395455+0000 front 2024-06-11T14:09:42.395511+0000 (oldest deadline 2024-06-11T14:10:07.095281+0000) 2024-06-11T14:10:08.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:08.201+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:08.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:08.201+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:08.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:08.275+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:08.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:08.275+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:09.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:08.603+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:09.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:08.603+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:09.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:08 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:08.603+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:42.395455+0000 front 2024-06-11T14:09:42.395511+0000 (oldest deadline 2024-06-11T14:10:07.095281+0000) 2024-06-11T14:10:09.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:09.170+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:09.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:09.170+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:09.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:09.274+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:09.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:09.274+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:09.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:09.274+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:10.047 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:09.562+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.393825+0000 front 2024-06-11T14:09:25.393619+0000 (oldest deadline 2024-06-11T14:09:51.293625+0000) 2024-06-11T14:10:10.048 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:09.562+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:36.495262+0000 front 2024-06-11T14:09:36.495169+0000 (oldest deadline 2024-06-11T14:10:02.394995+0000) 2024-06-11T14:10:10.048 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:09 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5[60837]: 2024-06-11T14:10:09.562+0000 7f21c5309640 -1 osd.5 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:42.395455+0000 front 2024-06-11T14:09:42.395511+0000 (oldest deadline 2024-06-11T14:10:07.095281+0000) 2024-06-11T14:10:10.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:10 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:10.185+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:10.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:10 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:10.185+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:10.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:10 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:10.239+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:10.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:10 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:10.239+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:10.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:10 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:10.239+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:10.919 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:10 smithi135 podman[78819]: 2024-06-11 14:10:10.643153744 +0000 UTC m=+5.553327129 container died 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, CEPH_POINT_RELEASE=, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.vendor=CentOS) 2024-06-11T14:10:11.186 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78819]: 2024-06-11 14:10:11.017351076 +0000 UTC m=+5.927524483 container cleanup 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, RELEASE=squid-3f51ea3, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-06-11T14:10:11.186 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 bash[78819]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5 2024-06-11T14:10:11.547 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78835]: 2024-06-11 14:10:11.285981903 +0000 UTC m=+0.638698506 container remove 99a4b288fdf14ad07a81f6134fbf603524f1297f8078a8845fca7d41f39d5e96 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-3f51ea3, org.label-schema.vendor=CentOS, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528) 2024-06-11T14:10:11.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:11.209+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:11.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:11.209+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:11.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:11.209+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:11.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:11.186+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:11.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:11.186+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:11.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:11 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:11.186+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:11.956 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78925]: 2024-06-11 14:10:11.542508505 +0000 UTC m=+0.017030547 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:10:11.956 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78925]: 2024-06-11 14:10:11.662027579 +0000 UTC m=+0.136549668 container create 2c659b3dbc7489af303e955f70cf192f17aa26e75225fbcf63f0c8773c796c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-3f51ea3, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:10:11.956 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78925]: 2024-06-11 14:10:11.87341795 +0000 UTC m=+0.347939993 container init 2c659b3dbc7489af303e955f70cf192f17aa26e75225fbcf63f0c8773c796c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-06-11T14:10:11.956 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78925]: 2024-06-11 14:10:11.876774331 +0000 UTC m=+0.351296376 container start 2c659b3dbc7489af303e955f70cf192f17aa26e75225fbcf63f0c8773c796c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-3f51ea3, ceph=True, io.buildah.version=1.35.3, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528) 2024-06-11T14:10:11.956 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:11 smithi135 podman[78925]: 2024-06-11 14:10:11.956785297 +0000 UTC m=+0.431307348 container attach 2c659b3dbc7489af303e955f70cf192f17aa26e75225fbcf63f0c8773c796c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240528, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-06-11T14:10:12.297 INFO:journalctl@ceph.osd.5.smithi135.stdout:Jun 11 14:10:12 smithi135 podman[78925]: 2024-06-11 14:10:12.133390468 +0000 UTC m=+0.607912511 container died 2c659b3dbc7489af303e955f70cf192f17aa26e75225fbcf63f0c8773c796c55 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240528, io.buildah.version=1.35.3, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD) 2024-06-11T14:10:12.298 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:12.194+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:12.298 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:12.194+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:12.298 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:12.194+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:12.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:12.199+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:12.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:12.199+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:12.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:12 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:12.199+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:12.772 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.5.service' 2024-06-11T14:10:12.812 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:10:12.813 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-06-11T14:10:12.813 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-06-11T14:10:12.813 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.6 2024-06-11T14:10:13.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:12 smithi135 systemd[1]: Stopping Ceph osd.6 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:10:13.547 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:13.147+0000 7fdb3bffd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:13.147+0000 7fdb3bffd640 -1 osd.6 128 *** Got signal Terminated *** 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:13.147+0000 7fdb3bffd640 -1 osd.6 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:13.148+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:13.148+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:13.148+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:13.211+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:13.211+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:13.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:13 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:13.211+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:14.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:14.126+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:14.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:14.126+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:14.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:14.126+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:14.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:14.203+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:14.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:14.203+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:14.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:14 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:14.203+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:15.367 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:15.105+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:15.367 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:15.105+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:15.368 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:15.105+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:15.368 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:15.235+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:15.368 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:15.236+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:15.368 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:15.236+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:15.368 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:15 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:15.236+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:16.105+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:16.105+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:16.105+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:16.258+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:16.258+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:16.258+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:16.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:16 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:16.258+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:17.133+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:17.133+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:17.133+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:17.267+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:17.267+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:17.267+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:17.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:17 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:17.267+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:18.488 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:18.104+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:25.465007+0000 front 2024-06-11T14:09:25.465049+0000 (oldest deadline 2024-06-11T14:09:53.065281+0000) 2024-06-11T14:10:18.488 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:18.104+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:37.066660+0000 front 2024-06-11T14:09:37.066758+0000 (oldest deadline 2024-06-11T14:10:04.067012+0000) 2024-06-11T14:10:18.488 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6[63097]: 2024-06-11T14:10:18.104+0000 7fdb3460b640 -1 osd.6 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:44.067471+0000 front 2024-06-11T14:09:44.067460+0000 (oldest deadline 2024-06-11T14:10:08.767411+0000) 2024-06-11T14:10:18.488 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:18.263+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:18.489 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:18.263+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:18.489 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:18.263+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:18.489 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:18 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:18.263+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:18.740 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:18 smithi135 podman[79031]: 2024-06-11 14:10:18.488648774 +0000 UTC m=+5.452410501 container died 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240528) 2024-06-11T14:10:19.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:18 smithi135 podman[79031]: 2024-06-11 14:10:18.890406968 +0000 UTC m=+5.854168716 container cleanup 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-3f51ea3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.35.3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_CLEAN=True) 2024-06-11T14:10:19.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:18 smithi135 bash[79031]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6 2024-06-11T14:10:19.467 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79047]: 2024-06-11 14:10:19.133991581 +0000 UTC m=+0.640455178 container remove 016c00269b62946461f906f3797a01ec8f9bafb6645fa00e5cd30c5b8a959921 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6, org.label-schema.build-date=20240528, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-06-11T14:10:19.467 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:19 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:19.227+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:19.467 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:19 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:19.227+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:19.467 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:19 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:19.227+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:19.468 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:19 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:19.227+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:19.778 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79135]: 2024-06-11 14:10:19.370457484 +0000 UTC m=+0.017190387 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:10:19.779 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79135]: 2024-06-11 14:10:19.501567351 +0000 UTC m=+0.148300238 container create c1bf4d3bf7dfb9bbe98345b8fae7d51f4dd46ec2b3d632f531c659ec5bfeb88d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.build-date=20240528, CEPH_POINT_RELEASE=) 2024-06-11T14:10:19.779 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79135]: 2024-06-11 14:10:19.704862263 +0000 UTC m=+0.351595158 container init c1bf4d3bf7dfb9bbe98345b8fae7d51f4dd46ec2b3d632f531c659ec5bfeb88d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-3f51ea3, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:10:19.779 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79135]: 2024-06-11 14:10:19.708098805 +0000 UTC m=+0.354831702 container start c1bf4d3bf7dfb9bbe98345b8fae7d51f4dd46ec2b3d632f531c659ec5bfeb88d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3) 2024-06-11T14:10:19.779 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79135]: 2024-06-11 14:10:19.779823125 +0000 UTC m=+0.426556024 container attach c1bf4d3bf7dfb9bbe98345b8fae7d51f4dd46ec2b3d632f531c659ec5bfeb88d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.35.3, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-06-11T14:10:20.047 INFO:journalctl@ceph.osd.6.smithi135.stdout:Jun 11 14:10:19 smithi135 podman[79135]: 2024-06-11 14:10:19.973931169 +0000 UTC m=+0.620664066 container died c1bf4d3bf7dfb9bbe98345b8fae7d51f4dd46ec2b3d632f531c659ec5bfeb88d (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.35.3, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240528) 2024-06-11T14:10:20.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:20 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:20.237+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:20.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:20 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:20.237+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:20.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:20 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:20.237+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:20.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:20 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:20.237+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:20.651 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.6.service' 2024-06-11T14:10:20.690 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:10:20.690 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-06-11T14:10:20.690 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-06-11T14:10:20.690 DEBUG:teuthology.orchestra.run.smithi135:> sudo systemctl stop ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.7 2024-06-11T14:10:21.037 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:20 smithi135 systemd[1]: Stopping Ceph osd.7 for 5368e33c-27f7-11ef-bc9b-c7b262605968... 2024-06-11T14:10:21.037 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.037+0000 7f0a8fc03640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-06-11T14:10:21.037 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.037+0000 7f0a8fc03640 -1 osd.7 128 *** Got signal Terminated *** 2024-06-11T14:10:21.037 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.037+0000 7f0a8fc03640 -1 osd.7 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-06-11T14:10:21.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.214+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:21.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.214+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:21.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.214+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:21.298 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:21 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:21.214+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:22.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:22 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:22.250+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:22.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:22 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:22.250+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:22.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:22 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:22.250+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:22.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:22 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:22.250+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:23.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:23.287+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:23.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:23.287+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:23.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:23.287+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:23.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:23 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:23.287+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:24.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:24 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:24.281+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:24.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:24 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:24.281+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:24.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:24 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:24.281+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:24.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:24 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:24.281+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:24.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:24 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:24.281+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.135:6804 osd.4 since back 2024-06-11T14:10:00.587306+0000 front 2024-06-11T14:10:00.587250+0000 (oldest deadline 2024-06-11T14:10:23.487161+0000) 2024-06-11T14:10:25.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:25 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:25.280+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6806 osd.0 since back 2024-06-11T14:09:26.882717+0000 front 2024-06-11T14:09:26.882817+0000 (oldest deadline 2024-06-11T14:09:50.982668+0000) 2024-06-11T14:10:25.547 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:25 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:25.280+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6816 osd.1 since back 2024-06-11T14:09:32.683448+0000 front 2024-06-11T14:09:32.683413+0000 (oldest deadline 2024-06-11T14:09:58.583534+0000) 2024-06-11T14:10:25.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:25 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:25.280+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6822 osd.2 since back 2024-06-11T14:09:45.985739+0000 front 2024-06-11T14:09:45.985789+0000 (oldest deadline 2024-06-11T14:10:10.586088+0000) 2024-06-11T14:10:25.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:25 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:25.280+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.69:6830 osd.3 since back 2024-06-11T14:09:50.586570+0000 front 2024-06-11T14:09:50.586506+0000 (oldest deadline 2024-06-11T14:10:14.686519+0000) 2024-06-11T14:10:25.548 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:25 smithi135 ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7[65349]: 2024-06-11T14:10:25.280+0000 7f0a88211640 -1 osd.7 128 heartbeat_check: no reply from 172.21.15.135:6804 osd.4 since back 2024-06-11T14:10:00.587306+0000 front 2024-06-11T14:10:00.587250+0000 (oldest deadline 2024-06-11T14:10:23.487161+0000) 2024-06-11T14:10:26.770 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:26 smithi135 podman[79237]: 2024-06-11 14:10:26.461077184 +0000 UTC m=+5.544995340 container died 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, io.buildah.version=1.35.3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-3f51ea3, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240528) 2024-06-11T14:10:26.770 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:26 smithi135 podman[79237]: 2024-06-11 14:10:26.771231111 +0000 UTC m=+5.855149274 container cleanup 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-3f51ea3, GIT_CLEAN=True, io.buildah.version=1.35.3, org.label-schema.build-date=20240528, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:10:27.047 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:26 smithi135 bash[79237]: ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7 2024-06-11T14:10:27.047 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:26 smithi135 podman[79299]: 2024-06-11 14:10:26.997731799 +0000 UTC m=+0.532755366 container remove 02983e37cce55e2f3aeeb4d88723c594feb41c545e01bcfa93d497c090295a19 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-3f51ea3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528) 2024-06-11T14:10:27.592 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:27 smithi135 podman[79384]: 2024-06-11 14:10:27.244382095 +0000 UTC m=+0.017761543 image pull 2d7459ec65a098c35c774154617f2893b0ad051f56ce89b72ab795f65a97fc22 quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310 2024-06-11T14:10:27.592 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:27 smithi135 podman[79384]: 2024-06-11 14:10:27.373810024 +0000 UTC m=+0.147189448 container create 067ee2cb7f2541b419d0fbb9fe31041f0ba3506dcb2ad6062de7ccf71a796ca0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.35.3, ceph=True, org.label-schema.build-date=20240528, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_BRANCH=HEAD) 2024-06-11T14:10:27.592 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:27 smithi135 podman[79384]: 2024-06-11 14:10:27.59323577 +0000 UTC m=+0.366615217 container init 067ee2cb7f2541b419d0fbb9fe31041f0ba3506dcb2ad6062de7ccf71a796ca0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240528, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47) 2024-06-11T14:10:27.851 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:27 smithi135 podman[79384]: 2024-06-11 14:10:27.596834566 +0000 UTC m=+0.370214005 container start 067ee2cb7f2541b419d0fbb9fe31041f0ba3506dcb2ad6062de7ccf71a796ca0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, io.buildah.version=1.35.3, RELEASE=squid-3f51ea3, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-06-11T14:10:27.851 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:27 smithi135 podman[79384]: 2024-06-11 14:10:27.668394469 +0000 UTC m=+0.441773913 container attach 067ee2cb7f2541b419d0fbb9fe31041f0ba3506dcb2ad6062de7ccf71a796ca0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.35.3, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240528, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, RELEASE=squid-3f51ea3, GIT_BRANCH=HEAD) 2024-06-11T14:10:27.851 INFO:journalctl@ceph.osd.7.smithi135.stdout:Jun 11 14:10:27 smithi135 podman[79384]: 2024-06-11 14:10:27.851946397 +0000 UTC m=+0.625325838 container died 067ee2cb7f2541b419d0fbb9fe31041f0ba3506dcb2ad6062de7ccf71a796ca0 (image=quay.ceph.io/ceph-ci/ceph@sha256:7cce454e59f5f61a0030b01fb0a6f9012c83522b7e7d1b60b63c90c2a73fa310, name=ceph-5368e33c-27f7-11ef-bc9b-c7b262605968-osd-7-deactivate, GIT_COMMIT=4a8012357b057f80e264ea5a6c36c04375a85d47, io.buildah.version=1.35.3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240528, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-3f51ea3, CEPH_POINT_RELEASE=) 2024-06-11T14:10:28.540 DEBUG:teuthology.orchestra.run.smithi135:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5368e33c-27f7-11ef-bc9b-c7b262605968@osd.7.service' 2024-06-11T14:10:28.578 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-06-11T14:10:28.579 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-06-11T14:10:28.579 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 --force --keep-logs 2024-06-11T14:13:24.866 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 --force --keep-logs 2024-06-11T14:13:45.452 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-11T14:13:45.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-06-11T14:13:45.515 INFO:tasks.cephadm:Archiving crash dumps... 2024-06-11T14:13:45.517 DEBUG:teuthology.misc:Transferring archived files from smithi069:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/remote/smithi069/crash 2024-06-11T14:13:45.518 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/crash -- . 2024-06-11T14:13:45.570 INFO:teuthology.orchestra.run.smithi069.stderr:tar: /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/crash: Cannot open: No such file or directory 2024-06-11T14:13:45.570 INFO:teuthology.orchestra.run.smithi069.stderr:tar: Error is not recoverable: exiting now 2024-06-11T14:13:45.572 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/remote/smithi135/crash 2024-06-11T14:13:45.573 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/crash -- . 2024-06-11T14:13:45.606 INFO:teuthology.orchestra.run.smithi135.stderr:tar: /var/lib/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/crash: Cannot open: No such file or directory 2024-06-11T14:13:45.606 INFO:teuthology.orchestra.run.smithi135.stderr:tar: Error is not recoverable: exiting now 2024-06-11T14:13:45.607 INFO:tasks.cephadm:Checking cluster log for badness... 2024-06-11T14:13:45.607 DEBUG:teuthology.orchestra.run.smithi069:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-11T14:13:45.641 INFO:teuthology.orchestra.run.smithi069.stdout:2024-06-11T13:45:24.818088+0000 mon.smithi069 (mon.0) 745 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-11T14:13:45.642 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-06-11T14:13:45.642 DEBUG:teuthology.orchestra.run.smithi069:> sudo egrep '\[SEC\]' /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-11T14:13:45.711 DEBUG:teuthology.orchestra.run.smithi069:> sudo egrep '\[ERR\]' /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-11T14:13:45.780 DEBUG:teuthology.orchestra.run.smithi069:> sudo egrep '\[WRN\]' /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-06-11T14:13:45.847 INFO:teuthology.orchestra.run.smithi069.stdout:2024-06-11T13:45:24.818088+0000 mon.smithi069 (mon.0) 745 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED) 2024-06-11T14:13:45.847 INFO:tasks.cephadm:Compressing logs... 2024-06-11T14:13:45.847 DEBUG:teuthology.orchestra.run.smithi069:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-11T14:13:45.890 DEBUG:teuthology.orchestra.run.smithi135:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-06-11T14:13:45.912 INFO:teuthology.orchestra.run.smithi069.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log 2024-06-11T14:13:45.912 INFO:teuthology.orchestra.run.smithi069.stderr:: No such file or directory 2024-06-11T14:13:45.912 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mon.smithi069.log 2024-06-11T14:13:45.913 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.2.log 2024-06-11T14:13:45.913 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mon.smithi069.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.audit.log 2024-06-11T14:13:45.913 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.3.log 2024-06-11T14:13:45.916 INFO:teuthology.orchestra.run.smithi135.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log 2024-06-11T14:13:45.916 INFO:teuthology.orchestra.run.smithi135.stderr:: No such file or directory 2024-06-11T14:13:45.916 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.audit.log 2024-06-11T14:13:45.916 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.7.log 2024-06-11T14:13:45.916 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log: /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.5.log 2024-06-11T14:13:45.916 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.4.log 2024-06-11T14:13:45.917 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mgr.smithi069.iruzyw.log 2024-06-11T14:13:45.917 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.1.log 2024-06-11T14:13:45.917 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mgr.smithi069.iruzyw.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.0.log 2024-06-11T14:13:45.917 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.cephadm.log 2024-06-11T14:13:45.918 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.0.log: 88.4% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log.gz 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.6.log 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.5.log: /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.cephadm.log 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi135.wtrawd.log 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mgr.smithi135.xcould.log 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi135.wtrawd.log: 85.8% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.cephadm.log.gz 87.5% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi135.wtrawd.log.gz 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr: 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mon.smithi135.log 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mgr.smithi135.xcould.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-volume.log 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr: 88.4% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.log.gz 2024-06-11T14:13:45.922 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mon.smithi135.log: gzip -5 91.8% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.audit.log.gz 2024-06-11T14:13:45.923 INFO:teuthology.orchestra.run.smithi135.stderr: --verbose -- /var/log/ceph/cephadm.log 2024-06-11T14:13:45.923 INFO:teuthology.orchestra.run.smithi069.stderr: 91.7% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.audit.log.gz 2024-06-11T14:13:45.927 INFO:teuthology.orchestra.run.smithi135.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 90.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-11T14:13:45.930 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi069.nsgdqs.log 2024-06-11T14:13:45.930 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-volume.log 2024-06-11T14:13:45.931 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi069.nsgdqs.log: 85.9% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph.cephadm.log.gz 88.2% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mds.foofs.smithi069.nsgdqs.log.gz 2024-06-11T14:13:45.931 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T14:13:45.934 INFO:teuthology.orchestra.run.smithi135.stderr: 91.2% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-volume.log.gz 2024-06-11T14:13:45.943 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-06-11T14:13:45.960 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-volume.log.gz 2024-06-11T14:13:45.963 INFO:teuthology.orchestra.run.smithi135.stderr: 90.0% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mgr.smithi135.xcould.log.gz 2024-06-11T14:13:45.980 INFO:teuthology.orchestra.run.smithi069.stderr: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-06-11T14:13:46.274 INFO:teuthology.orchestra.run.smithi135.stderr: 92.7% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mon.smithi135.log.gz 2024-06-11T14:13:46.400 INFO:teuthology.orchestra.run.smithi069.stderr: 89.3% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mgr.smithi069.iruzyw.log.gz 2024-06-11T14:13:48.077 INFO:teuthology.orchestra.run.smithi069.stderr: 91.6% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-mon.smithi069.log.gz 2024-06-11T14:13:54.321 INFO:teuthology.orchestra.run.smithi069.stderr: 94.5% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.2.log.gz 2024-06-11T14:13:58.633 INFO:teuthology.orchestra.run.smithi135.stderr: 93.9% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.4.log.gz 2024-06-11T14:13:58.954 INFO:teuthology.orchestra.run.smithi135.stderr: 94.2% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.6.log.gz 2024-06-11T14:13:59.317 INFO:teuthology.orchestra.run.smithi069.stderr: 94.1% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.0.log.gz 2024-06-11T14:13:59.724 INFO:teuthology.orchestra.run.smithi069.stderr: 94.2% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.1.log.gz 2024-06-11T14:14:00.169 INFO:teuthology.orchestra.run.smithi069.stderr: 94.1% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.3.log.gz 2024-06-11T14:14:00.171 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-06-11T14:14:00.172 INFO:teuthology.orchestra.run.smithi069.stderr:real 0m14.269s 2024-06-11T14:14:00.172 INFO:teuthology.orchestra.run.smithi069.stderr:user 0m50.530s 2024-06-11T14:14:00.172 INFO:teuthology.orchestra.run.smithi069.stderr:sys 0m1.983s 2024-06-11T14:14:02.127 INFO:teuthology.orchestra.run.smithi135.stderr: 94.1% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.5.log.gz 2024-06-11T14:14:05.744 INFO:teuthology.orchestra.run.smithi135.stderr: 94.1% -- replaced with /var/log/ceph/5368e33c-27f7-11ef-bc9b-c7b262605968/ceph-osd.7.log.gz 2024-06-11T14:14:05.746 INFO:teuthology.orchestra.run.smithi135.stderr: 2024-06-11T14:14:05.746 INFO:teuthology.orchestra.run.smithi135.stderr:real 0m19.842s 2024-06-11T14:14:05.747 INFO:teuthology.orchestra.run.smithi135.stderr:user 0m59.849s 2024-06-11T14:14:05.747 INFO:teuthology.orchestra.run.smithi135.stderr:sys 0m2.284s 2024-06-11T14:14:05.747 INFO:tasks.cephadm:Archiving logs... 2024-06-11T14:14:05.747 DEBUG:teuthology.misc:Transferring archived files from smithi069:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/remote/smithi069/log 2024-06-11T14:14:05.748 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-11T14:14:12.773 DEBUG:teuthology.misc:Transferring archived files from smithi135:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/remote/smithi135/log 2024-06-11T14:14:12.774 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /var/log/ceph -- . 2024-06-11T14:14:21.306 INFO:tasks.cephadm:Removing cluster... 2024-06-11T14:14:21.307 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 --force 2024-06-11T14:14:21.652 DEBUG:teuthology.orchestra.run.smithi135:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5368e33c-27f7-11ef-bc9b-c7b262605968 --force 2024-06-11T14:14:22.109 INFO:tasks.cephadm:Removing cephadm ... 2024-06-11T14:14:22.109 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-11T14:14:22.125 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf /home/ubuntu/cephtest/cephadm 2024-06-11T14:14:22.141 INFO:tasks.cephadm:Teardown complete 2024-06-11T14:14:22.141 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-06-11T14:14:22.152 INFO:teuthology.task.clock:Checking final clock skew... 2024-06-11T14:14:22.152 DEBUG:teuthology.orchestra.run.smithi069:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-11T14:14:22.168 DEBUG:teuthology.orchestra.run.smithi135:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-06-11T14:14:22.180 INFO:teuthology.orchestra.run.smithi069.stderr:bash: line 1: ntpq: command not found 2024-06-11T14:14:22.186 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-11T14:14:22.186 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2024-06-11T14:14:22.186 INFO:teuthology.orchestra.run.smithi069.stdout:^* hv01.front.sepia.ceph.com 3 6 377 63 +911us[ +977us] +/- 45ms 2024-06-11T14:14:22.186 INFO:teuthology.orchestra.run.smithi069.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 59 -2215us[-2215us] +/- 73ms 2024-06-11T14:14:22.186 INFO:teuthology.orchestra.run.smithi069.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 61 +680us[ +680us] +/- 71ms 2024-06-11T14:14:22.186 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T14:14:22.194 INFO:teuthology.orchestra.run.smithi135.stderr:bash: line 1: ntpq: command not found 2024-06-11T14:14:22.199 INFO:teuthology.orchestra.run.smithi135.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-06-11T14:14:22.199 INFO:teuthology.orchestra.run.smithi135.stdout:=============================================================================== 2024-06-11T14:14:22.199 INFO:teuthology.orchestra.run.smithi135.stdout:^* hv01.front.sepia.ceph.com 3 6 377 59 +919us[ +930us] +/- 45ms 2024-06-11T14:14:22.199 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 58 -2219us[-2219us] +/- 73ms 2024-06-11T14:14:22.199 INFO:teuthology.orchestra.run.smithi135.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 57 +685us[ +685us] +/- 71ms 2024-06-11T14:14:22.199 INFO:teuthology.orchestra.run.smithi135.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-06-11T14:14:22.200 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-06-11T14:14:22.209 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-06-11T14:14:22.210 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-06-11T14:14:22.226 DEBUG:teuthology.orchestra.run.smithi069:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-11T14:14:22.306 DEBUG:teuthology.orchestra.run.smithi135:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-06-11T14:14:22.380 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-11T14:14:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T14:14:22.409 DEBUG:teuthology.orchestra.run.smithi135:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-06-11T14:14:22.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T14:14:22.437 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-06-11T14:14:22.447 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-06-11T14:14:22.467 INFO:teuthology.task.internal:Duration was 2664.655522 seconds 2024-06-11T14:14:22.467 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-06-11T14:14:22.484 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-06-11T14:14:22.484 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-11T14:14:22.487 DEBUG:teuthology.orchestra.run.smithi135:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-06-11T14:14:22.523 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-11T14:14:22.526 INFO:teuthology.orchestra.run.smithi135.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-06-11T14:14:23.044 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-06-11T14:14:23.044 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi069.front.sepia.ceph.com 2024-06-11T14:14:23.044 DEBUG:teuthology.orchestra.run.smithi069:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-11T14:14:23.105 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi135.front.sepia.ceph.com 2024-06-11T14:14:23.105 DEBUG:teuthology.orchestra.run.smithi135:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-06-11T14:14:23.136 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-06-11T14:14:23.136 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-11T14:14:23.148 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-06-11T14:14:23.205 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-06-11T14:14:23.206 DEBUG:teuthology.orchestra.run.smithi069:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-11T14:14:23.216 DEBUG:teuthology.orchestra.run.smithi135:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-06-11T14:14:37.076 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-06-11T14:14:37.088 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-06-11T14:14:37.088 DEBUG:teuthology.orchestra.run.smithi069:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-11T14:14:37.113 DEBUG:teuthology.orchestra.run.smithi135:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-06-11T14:14:37.143 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-06-11T14:14:37.153 DEBUG:teuthology.orchestra.run.smithi069:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-11T14:14:37.155 DEBUG:teuthology.orchestra.run.smithi135:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-06-11T14:14:37.176 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = core 2024-06-11T14:14:37.209 INFO:teuthology.orchestra.run.smithi135.stdout:kernel.core_pattern = core 2024-06-11T14:14:37.529 DEBUG:teuthology.orchestra.run.smithi069:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-11T14:14:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T14:14:37.545 DEBUG:teuthology.orchestra.run.smithi135:> test -e /home/ubuntu/cephtest/archive/coredump 2024-06-11T14:14:37.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-06-11T14:14:37.588 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-06-11T14:14:37.598 INFO:teuthology.task.internal:Transferring archived files... 2024-06-11T14:14:37.599 DEBUG:teuthology.misc:Transferring archived files from smithi069:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/remote/smithi069 2024-06-11T14:14:37.600 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-11T14:14:37.669 DEBUG:teuthology.misc:Transferring archived files from smithi135:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-06-10_21:08:02-orch-squid-distro-default-smithi/7749326/remote/smithi135 2024-06-11T14:14:37.669 DEBUG:teuthology.orchestra.run.smithi135:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-06-11T14:14:37.712 INFO:teuthology.task.internal:Removing archive directory... 2024-06-11T14:14:37.712 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-11T14:14:37.715 DEBUG:teuthology.orchestra.run.smithi135:> rm -rf -- /home/ubuntu/cephtest/archive 2024-06-11T14:14:37.764 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-06-11T14:14:37.775 INFO:teuthology.task.internal:Not uploading archives. 2024-06-11T14:14:37.775 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-06-11T14:14:37.793 INFO:teuthology.task.internal:Tidying up after the test... 2024-06-11T14:14:37.793 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-11T14:14:37.796 DEBUG:teuthology.orchestra.run.smithi135:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-06-11T14:14:37.809 INFO:teuthology.orchestra.run.smithi069.stdout: 265786 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 11 14:14 /home/ubuntu/cephtest 2024-06-11T14:14:37.821 INFO:teuthology.orchestra.run.smithi135.stdout: 265316 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 11 14:14 /home/ubuntu/cephtest 2024-06-11T14:14:37.822 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-06-11T14:14:37.832 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-06-11T14:14:37.868 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2664.6555223464966 failure_reason: '"2024-06-11T13:45:24.818088+0000 mon.smithi069 (mon.0) 745 : cluster [WRN] Health check failed: failed to probe daemons or devices (CEPHADM_REFRESH_FAILED)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-06-11T14:14:37.869 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-06-11T14:14:37.937 INFO:teuthology.run:FAIL