2024-04-12T22:46:52.185 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-12T22:46:52.186 DEBUG:teuthology.run:Teuthology command: teuthology --archive /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631 --owner scheduled_teuthology@teuthology --name teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi --description orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose -- /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/orig.config.yaml 2024-04-12T22:46:52.223 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T22:46:52.329 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7654631' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: 98423fadeecbba811088c3136f8295dae6dfe870 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 2723 sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a sleep_before_teardown: 0 subset: 24/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_98423fadeecbba811088c3136f8295dae6dfe870/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 98423fadeecbba811088c3136f8295dae6dfe870 targets: smithi144.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNdaf8Xngm/oFVwUNyqQnojKjbkKNh5z28+XuesW+Jyzli78mid3XcT9Rj+Qdtvane9+STVG0INvO5lthV1T0yw= smithi181.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEgMIC+qb4bkv4MhnV+yPCvTI1sRLxyEXDczVoeRf4gCPRqTFLnpOZ5YNIxrugSU1dg7YKqG7kxBl877Suu7z/I= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-12_21:08:02 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-12T22:46:52.329 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_98423fadeecbba811088c3136f8295dae6dfe870/qa; will attempt to use it 2024-04-12T22:46:52.330 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_98423fadeecbba811088c3136f8295dae6dfe870/qa/tasks 2024-04-12T22:46:52.330 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-12T22:46:52.332 INFO:teuthology.task.internal:Checking packages... 2024-04-12T22:46:52.356 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a' 2024-04-12T22:46:52.356 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-12T22:46:52.356 INFO:teuthology.packaging:ref: None 2024-04-12T22:46:52.356 INFO:teuthology.packaging:tag: None 2024-04-12T22:46:52.356 INFO:teuthology.packaging:branch: squid 2024-04-12T22:46:52.356 INFO:teuthology.packaging:sha1: b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T22:46:52.356 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-12T22:46:52.577 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2112.g98423fad 2024-04-12T22:46:52.578 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-12T22:46:52.587 INFO:teuthology.task.internal:no buildpackages task found 2024-04-12T22:46:52.587 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-12T22:46:52.654 INFO:teuthology.task.internal:Saving configuration 2024-04-12T22:46:52.668 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-12T22:46:52.698 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T22:46:52.720 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi144.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 22:41:50.914611', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNdaf8Xngm/oFVwUNyqQnojKjbkKNh5z28+XuesW+Jyzli78mid3XcT9Rj+Qdtvane9+STVG0INvO5lthV1T0yw='} 2024-04-12T22:46:52.739 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 22:41:50.915565', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEgMIC+qb4bkv4MhnV+yPCvTI1sRLxyEXDczVoeRf4gCPRqTFLnpOZ5YNIxrugSU1dg7YKqG7kxBl877Suu7z/I='} 2024-04-12T22:46:52.739 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-12T22:46:52.752 INFO:teuthology.task.internal:roles: ubuntu@smithi144.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-12T22:46:52.752 INFO:teuthology.task.internal:roles: ubuntu@smithi181.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-12T22:46:52.752 INFO:teuthology.run_tasks:Running task console_log... 2024-04-12T22:46:52.981 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9addfcdf70>, signals=[15]) 2024-04-12T22:46:52.981 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-12T22:46:52.992 INFO:teuthology.task.internal:Opening connections... 2024-04-12T22:46:52.993 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T22:46:52.994 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:46:53.085 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi181.front.sepia.ceph.com 2024-04-12T22:46:53.086 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:46:53.175 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-12T22:46:53.186 DEBUG:teuthology.orchestra.run.smithi144:> uname -m 2024-04-12T22:46:53.207 INFO:teuthology.orchestra.run.smithi144.stdout:x86_64 2024-04-12T22:46:53.207 DEBUG:teuthology.orchestra.run.smithi144:> cat /etc/os-release 2024-04-12T22:46:53.262 INFO:teuthology.orchestra.run.smithi144.stdout:NAME="CentOS Stream" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:VERSION="9" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:ID="centos" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:ID_LIKE="rhel fedora" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:VERSION_ID="9" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:PLATFORM_ID="platform:el9" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:ANSI_COLOR="0;31" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:LOGO="fedora-logo-icon" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:HOME_URL="https://centos.org/" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T22:46:53.263 INFO:teuthology.orchestra.run.smithi144.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T22:46:53.264 INFO:teuthology.lock.ops:Updating smithi144.front.sepia.ceph.com on lock server 2024-04-12T22:46:53.288 DEBUG:teuthology.orchestra.run.smithi181:> uname -m 2024-04-12T22:46:53.305 INFO:teuthology.orchestra.run.smithi181.stdout:x86_64 2024-04-12T22:46:53.305 DEBUG:teuthology.orchestra.run.smithi181:> cat /etc/os-release 2024-04-12T22:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout:NAME="CentOS Stream" 2024-04-12T22:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION="9" 2024-04-12T22:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout:ID="centos" 2024-04-12T22:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout:ID_LIKE="rhel fedora" 2024-04-12T22:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_ID="9" 2024-04-12T22:46:53.359 INFO:teuthology.orchestra.run.smithi181.stdout:PLATFORM_ID="platform:el9" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:ANSI_COLOR="0;31" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:LOGO="fedora-logo-icon" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:HOME_URL="https://centos.org/" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-12T22:46:53.360 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-12T22:46:53.360 INFO:teuthology.lock.ops:Updating smithi181.front.sepia.ceph.com on lock server 2024-04-12T22:46:53.379 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-12T22:46:53.394 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-12T22:46:53.462 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-12T22:46:53.463 DEBUG:teuthology.orchestra.run.smithi144:> test '!' -e /home/ubuntu/cephtest 2024-04-12T22:46:53.465 DEBUG:teuthology.orchestra.run.smithi181:> test '!' -e /home/ubuntu/cephtest 2024-04-12T22:46:53.481 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-12T22:46:53.523 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-12T22:46:53.523 DEBUG:teuthology.orchestra.run.smithi144:> test -z $(ls -A /var/lib/ceph) 2024-04-12T22:46:53.525 DEBUG:teuthology.orchestra.run.smithi181:> test -z $(ls -A /var/lib/ceph) 2024-04-12T22:46:53.550 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-12T22:46:53.649 INFO:teuthology.run_tasks:Running task kernel... 2024-04-12T22:46:53.668 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-12T22:46:53.668 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-12T22:46:53.669 DEBUG:teuthology.orchestra.run.smithi144:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T22:46:53.669 DEBUG:teuthology.orchestra.run.smithi181:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-12T22:46:53.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:46:53.686 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-04-12T22:46:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:46:53.687 DEBUG:teuthology.orchestra.run.smithi144:> uname -r 2024-04-12T22:46:53.740 INFO:teuthology.orchestra.run.smithi181.stdout:5.14.0-361.el9.x86_64 2024-04-12T22:46:53.740 INFO:teuthology.task.kernel:Running kernel on smithi181: 5.14.0-361.el9.x86_64 2024-04-12T22:46:53.740 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum install -y kernel 2024-04-12T22:46:53.742 INFO:teuthology.orchestra.run.smithi144.stdout:5.14.0-361.el9.x86_64 2024-04-12T22:46:53.742 INFO:teuthology.task.kernel:Running kernel on smithi144: 5.14.0-361.el9.x86_64 2024-04-12T22:46:53.742 DEBUG:teuthology.orchestra.run.smithi144:> sudo yum install -y kernel 2024-04-12T22:46:55.833 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS-9 - AppStream 16 MB/s | 8.1 MB 00:00 2024-04-12T22:46:55.896 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS-9 - AppStream 16 MB/s | 8.1 MB 00:00 2024-04-12T22:46:57.746 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS-9 - BaseOS 4.7 MB/s | 2.5 MB 00:00 2024-04-12T22:46:57.749 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS-9 - BaseOS 4.3 MB/s | 2.5 MB 00:00 2024-04-12T22:46:58.912 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-04-12T22:47:01.096 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - AppStream 17 MB/s | 19 MB 00:01 2024-04-12T22:47:02.906 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - BaseOS 1.7 MB/s | 8.0 MB 00:04 2024-04-12T22:47:04.932 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - AppStream 21 MB/s | 19 MB 00:00 2024-04-12T22:47:05.297 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - CRB 9.6 MB/s | 6.2 MB 00:00 2024-04-12T22:47:07.697 INFO:teuthology.orchestra.run.smithi144.stdout:CentOS Stream 9 - Extras packages 13 kB/s | 16 kB 00:01 2024-04-12T22:47:08.732 INFO:teuthology.orchestra.run.smithi144.stdout:Extra Packages for Enterprise Linux 24 MB/s | 21 MB 00:00 2024-04-12T22:47:09.141 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.2 MB 00:00 2024-04-12T22:47:10.625 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - Extras packages 54 kB/s | 16 kB 00:00 2024-04-12T22:47:11.877 INFO:teuthology.orchestra.run.smithi181.stdout:Extra Packages for Enterprise Linux 19 MB/s | 21 MB 00:01 2024-04-12T22:47:13.865 INFO:teuthology.orchestra.run.smithi144.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr:Modular dependency problems: 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 1: conflicting requests 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 2: conflicting requests 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 3: conflicting requests 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 4: conflicting requests 2024-04-12T22:47:16.107 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 5: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 6: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 7: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 8: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 9: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 10: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 11: conflicting requests 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.108 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 12: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 13: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 14: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 15: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 16: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 17: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 18: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 19: conflicting requests 2024-04-12T22:47:16.109 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 20: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 21: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 22: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 23: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 24: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 25: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 26: conflicting requests 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.110 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 27: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 28: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 29: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 30: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 31: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 32: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 33: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 34: conflicting requests 2024-04-12T22:47:16.111 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 35: conflicting requests 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 36: conflicting requests 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 37: conflicting requests 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 39: conflicting requests 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.112 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.113 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.114 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.115 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.115 INFO:teuthology.orchestra.run.smithi144.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T22:47:16.115 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.115 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.115 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.115 INFO:teuthology.orchestra.run.smithi144.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T22:47:16.522 INFO:teuthology.orchestra.run.smithi144.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T22:47:16.522 INFO:teuthology.orchestra.run.smithi144.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T22:47:16.628 INFO:teuthology.orchestra.run.smithi144.stdout:Dependencies resolved. 2024-04-12T22:47:16.636 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2024-04-12T22:47:16.636 INFO:teuthology.orchestra.run.smithi144.stdout: Package Architecture Version Repository Size 2024-04-12T22:47:16.636 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout:Installing: 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout:Installing dependencies: 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction Summary 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout:================================================================================ 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout:Install 4 Packages 2024-04-12T22:47:16.637 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:47:16.638 INFO:teuthology.orchestra.run.smithi144.stdout:Total download size: 77 M 2024-04-12T22:47:16.638 INFO:teuthology.orchestra.run.smithi144.stdout:Installed size: 123 M 2024-04-12T22:47:16.638 INFO:teuthology.orchestra.run.smithi144.stdout:Downloading Packages: 2024-04-12T22:47:17.068 INFO:teuthology.orchestra.run.smithi181.stdout:lab-extras 15 kB/s | 1.7 kB 00:00 2024-04-12T22:47:17.216 INFO:teuthology.orchestra.run.smithi144.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 694 kB/s | 214 kB 00:00 2024-04-12T22:47:19.141 INFO:teuthology.orchestra.run.smithi144.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 6.8 MB/s | 15 MB 00:02 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr:Modular dependency problems: 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 1: conflicting requests 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 2: conflicting requests 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 3: conflicting requests 2024-04-12T22:47:19.323 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 4: conflicting requests 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 5: conflicting requests 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 6: conflicting requests 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 7: conflicting requests 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 8: conflicting requests 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 9: conflicting requests 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.324 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 10: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 11: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 12: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 13: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 14: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 15: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 16: conflicting requests 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.325 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 17: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 18: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 19: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 20: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 21: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 22: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 23: conflicting requests 2024-04-12T22:47:19.326 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 24: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 25: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 26: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 27: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 28: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 29: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 30: conflicting requests 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.327 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 31: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 32: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 33: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 34: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 35: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 36: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 37: conflicting requests 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.328 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 38: cannot install the best candidate for the job 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 39: conflicting requests 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 40: cannot install the best candidate for the job 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.329 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 41: cannot install the best candidate for the job 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 42: cannot install the best candidate for the job 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 43: cannot install the best candidate for the job 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.330 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 44: cannot install the best candidate for the job 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: Problem 45: cannot install the best candidate for the job 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.331 INFO:teuthology.orchestra.run.smithi181.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-12T22:47:19.750 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-12T22:47:19.751 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-12T22:47:19.864 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-04-12T22:47:19.869 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-04-12T22:47:19.869 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout: kernel x86_64 5.14.0-435.el9 baseos 214 k 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-core x86_64 5.14.0-435.el9 baseos 15 M 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules x86_64 5.14.0-435.el9 baseos 34 M 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-core x86_64 5.14.0-435.el9 baseos 28 M 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:Install 4 Packages 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 77 M 2024-04-12T22:47:19.870 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 123 M 2024-04-12T22:47:19.871 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-04-12T22:47:20.194 INFO:teuthology.orchestra.run.smithi181.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 838 kB/s | 214 kB 00:00 2024-04-12T22:47:20.700 INFO:teuthology.orchestra.run.smithi144.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 8.9 MB/s | 34 MB 00:03 2024-04-12T22:47:20.975 INFO:teuthology.orchestra.run.smithi144.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 7.4 MB/s | 28 MB 00:03 2024-04-12T22:47:20.975 INFO:teuthology.orchestra.run.smithi144.stdout:-------------------------------------------------------------------------------- 2024-04-12T22:47:20.975 INFO:teuthology.orchestra.run.smithi144.stdout:Total 18 MB/s | 77 MB 00:04 2024-04-12T22:47:21.228 INFO:teuthology.orchestra.run.smithi181.stdout:(2/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 26 MB/s | 34 MB 00:01 2024-04-12T22:47:21.378 INFO:teuthology.orchestra.run.smithi181.stdout:(3/4): kernel-core-5.14.0-435.el9.x86_64.rpm 10 MB/s | 15 MB 00:01 2024-04-12T22:47:21.647 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction check 2024-04-12T22:47:22.025 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction check succeeded. 2024-04-12T22:47:22.025 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction test 2024-04-12T22:47:22.128 INFO:teuthology.orchestra.run.smithi181.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 14 MB/s | 28 MB 00:01 2024-04-12T22:47:22.131 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-04-12T22:47:22.131 INFO:teuthology.orchestra.run.smithi181.stdout:Total 34 MB/s | 77 MB 00:02 2024-04-12T22:47:22.805 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-04-12T22:47:23.019 INFO:teuthology.orchestra.run.smithi144.stdout:Transaction test succeeded. 2024-04-12T22:47:23.020 INFO:teuthology.orchestra.run.smithi144.stdout:Running transaction 2024-04-12T22:47:23.155 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-04-12T22:47:23.155 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-04-12T22:47:24.193 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-04-12T22:47:24.193 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-04-12T22:47:25.487 INFO:teuthology.orchestra.run.smithi144.stdout: Preparing : 1/1 2024-04-12T22:47:26.092 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T22:47:26.175 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T22:47:26.680 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-04-12T22:47:27.335 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-12T22:47:27.419 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T22:47:28.942 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T22:47:29.096 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T22:47:30.189 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T22:47:30.322 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T22:47:33.774 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T22:47:33.822 INFO:teuthology.orchestra.run.smithi144.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:47:35.034 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T22:47:35.073 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:47:38.562 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:47:39.808 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:07.671 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:07.672 INFO:teuthology.orchestra.run.smithi144.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T22:48:07.672 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:48:07.718 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:08.030 INFO:teuthology.orchestra.run.smithi144.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:08.030 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T22:48:08.030 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T22:48:08.030 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout:Installed: 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:48:08.460 INFO:teuthology.orchestra.run.smithi144.stdout:Complete! 2024-04-12T22:48:08.771 DEBUG:teuthology.orchestra.run.smithi144:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T22:48:08.917 INFO:teuthology.orchestra.run.smithi144.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T22:48:08.918 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T22:48:08.918 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T22:48:08.919 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T22:48:08.919 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T22:48:08.919 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-04-12T22:48:08.919 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi144.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T22:48:08.919 DEBUG:teuthology.orchestra.run.smithi144:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T22:48:08.959 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:08.959 INFO:teuthology.orchestra.run.smithi181.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-12T22:48:08.959 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:48:09.000 INFO:teuthology.orchestra.run.smithi144.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T22:48:09.000 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T22:48:09.001 DEBUG:teuthology.orchestra.run.smithi144:> sudo rpm -qi grub2-tools 2024-04-12T22:48:09.005 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:09.034 INFO:teuthology.orchestra.run.smithi144.stdout:Name : grub2-tools 2024-04-12T22:48:09.034 INFO:teuthology.orchestra.run.smithi144.stdout:Epoch : 1 2024-04-12T22:48:09.034 INFO:teuthology.orchestra.run.smithi144.stdout:Version : 2.06 2024-04-12T22:48:09.034 INFO:teuthology.orchestra.run.smithi144.stdout:Release : 68.el9 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Architecture: x86_64 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Group : Unspecified 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Size : 8264254 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:License : GPLv3+ 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Packager : builder@centos.org 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Vendor : CentOS 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Summary : Support tools for GRUB. 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout:Description : 2024-04-12T22:48:09.035 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:48:09.036 INFO:teuthology.orchestra.run.smithi144.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T22:48:09.036 INFO:teuthology.orchestra.run.smithi144.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T22:48:09.036 INFO:teuthology.orchestra.run.smithi144.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T22:48:09.036 INFO:teuthology.orchestra.run.smithi144.stdout:hardware devices. 2024-04-12T22:48:09.036 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:48:09.036 INFO:teuthology.orchestra.run.smithi144.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T22:48:09.037 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T22:48:09.037 INFO:teuthology.task.kernel:Updating grub on smithi144 to boot 5.14.0-435.el9.x86_64 2024-04-12T22:48:09.037 DEBUG:teuthology.orchestra.run.smithi144:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T22:48:09.381 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:09.381 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-12T22:48:09.381 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-12T22:48:09.381 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-12T22:48:09.653 INFO:teuthology.orchestra.run.smithi144.stderr:Generating grub configuration file ... 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:48:09.804 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-04-12T22:48:10.157 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T22:48:10.260 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T22:48:10.261 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T22:48:10.261 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-12T22:48:10.261 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-12T22:48:10.261 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-12T22:48:10.261 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-04-12T22:48:10.261 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi181.front.sepia.ceph.com, path=None, version=distro) 2024-04-12T22:48:10.261 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2024-04-12T22:48:10.342 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-12T22:48:10.342 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-12T22:48:10.342 DEBUG:teuthology.orchestra.run.smithi181:> sudo rpm -qi grub2-tools 2024-04-12T22:48:10.375 INFO:teuthology.orchestra.run.smithi181.stdout:Name : grub2-tools 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Epoch : 1 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Version : 2.06 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Release : 68.el9 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: x86_64 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Group : Unspecified 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Size : 8264254 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:License : GPLv3+ 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-12T22:48:10.376 INFO:teuthology.orchestra.run.smithi181.stdout:Packager : builder@centos.org 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:Vendor : CentOS 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:Summary : Support tools for GRUB. 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:Description : 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:hardware devices. 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:48:10.377 INFO:teuthology.orchestra.run.smithi181.stdout:This subpackage provides tools for support of all platforms. 2024-04-12T22:48:10.378 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-12T22:48:10.379 INFO:teuthology.task.kernel:Updating grub on smithi181 to boot 5.14.0-435.el9.x86_64 2024-04-12T22:48:10.379 DEBUG:teuthology.orchestra.run.smithi181:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-12T22:48:10.989 INFO:teuthology.orchestra.run.smithi144.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T22:48:11.015 INFO:teuthology.orchestra.run.smithi181.stderr:Generating grub configuration file ... 2024-04-12T22:48:11.030 INFO:teuthology.orchestra.run.smithi144.stderr:done 2024-04-12T22:48:11.033 DEBUG:teuthology.orchestra.run.smithi144:> mktemp 2024-04-12T22:48:11.053 INFO:teuthology.orchestra.run.smithi144.stdout:/tmp/tmp.4ZXTH2gjpv 2024-04-12T22:48:11.053 DEBUG:teuthology.orchestra.run.smithi144:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.4ZXTH2gjpv 2024-04-12T22:48:11.122 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 0666 /tmp/tmp.4ZXTH2gjpv 2024-04-12T22:48:11.261 DEBUG:teuthology.orchestra.remote:smithi144:/tmp/tmp.4ZXTH2gjpv is 6KB 2024-04-12T22:48:11.281 DEBUG:teuthology.orchestra.run.smithi144:> rm -fr /tmp/tmp.4ZXTH2gjpv 2024-04-12T22:48:11.296 DEBUG:teuthology.orchestra.run.smithi144:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T22:48:11.367 INFO:teuthology.orchestra.run.smithi144.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T22:48:11.368 INFO:teuthology.orchestra.run.smithi144.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T22:48:11.368 INFO:teuthology.orchestra.run.smithi144.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T22:48:11.368 INFO:teuthology.orchestra.run.smithi144.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T22:48:11.370 DEBUG:teuthology.orchestra.run.smithi144:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T22:48:11.535 DEBUG:teuthology.orchestra.run.smithi144:> sudo shutdown -r now 2024-04-12T22:48:12.416 INFO:teuthology.orchestra.run.smithi181.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-12T22:48:12.453 INFO:teuthology.orchestra.run.smithi181.stderr:done 2024-04-12T22:48:12.456 DEBUG:teuthology.orchestra.run.smithi181:> mktemp 2024-04-12T22:48:12.479 INFO:teuthology.orchestra.run.smithi181.stdout:/tmp/tmp.wriTnDkykN 2024-04-12T22:48:12.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wriTnDkykN 2024-04-12T22:48:12.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0666 /tmp/tmp.wriTnDkykN 2024-04-12T22:48:12.686 DEBUG:teuthology.orchestra.remote:smithi181:/tmp/tmp.wriTnDkykN is 6KB 2024-04-12T22:48:12.737 DEBUG:teuthology.orchestra.run.smithi181:> rm -fr /tmp/tmp.wriTnDkykN 2024-04-12T22:48:12.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo /bin/ls /boot/loader/entries || true 2024-04-12T22:48:12.818 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-12T22:48:12.818 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-12T22:48:12.818 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-12T22:48:12.818 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-12T22:48:12.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-12T22:48:12.970 DEBUG:teuthology.orchestra.run.smithi181:> sudo shutdown -r now 2024-04-12T22:48:13.044 ERROR:paramiko.transport:Socket exception: Connection reset by peer (104) 2024-04-12T22:48:41.566 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-04-12T22:48:41.567 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:48:41.568 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:48:42.974 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-04-12T22:48:42.974 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:48:42.975 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:48:59.934 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.144 2024-04-12T22:49:01.470 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-04-12T22:49:08.942 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:49:08.943 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:49:10.473 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:49:10.474 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:49:11.998 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.144 2024-04-12T22:49:13.534 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-04-12T22:49:24.008 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:49:24.009 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:49:25.536 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:49:25.537 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:49:42.430 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.144 2024-04-12T22:49:43.966 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-04-12T22:49:57.444 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:49:57.445 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:49:57.842 DEBUG:teuthology.orchestra.run.smithi144:> true 2024-04-12T22:49:58.375 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:49:58.375 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-435.el9.x86_64"... 2024-04-12T22:49:58.375 DEBUG:teuthology.orchestra.run.smithi144:> uname -r 2024-04-12T22:49:58.388 INFO:teuthology.orchestra.run.smithi144.stdout:5.14.0-435.el9.x86_64 2024-04-12T22:49:58.389 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T22:49:58.389 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T22:49:58.389 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T22:49:58.967 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:49:58.968 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:49:59.390 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-12T22:49:59.390 DEBUG:teuthology.orchestra.run.smithi144:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T22:49:59.396 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-04-12T22:49:59.487 INFO:teuthology.orchestra.run.smithi144.stdout:ttyS1 2024-04-12T22:49:59.519 DEBUG:teuthology.parallel:result is None 2024-04-12T22:49:59.753 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:49:59.753 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-435.el9.x86_64"... 2024-04-12T22:49:59.753 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-04-12T22:49:59.767 INFO:teuthology.orchestra.run.smithi181.stdout:5.14.0-435.el9.x86_64 2024-04-12T22:49:59.767 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-12T22:49:59.767 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-12T22:49:59.767 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-12T22:50:00.768 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-12T22:50:00.768 DEBUG:teuthology.orchestra.run.smithi181:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-12T22:50:00.841 INFO:teuthology.orchestra.run.smithi181.stdout:ttyS1 2024-04-12T22:50:00.879 DEBUG:teuthology.parallel:result is None 2024-04-12T22:50:00.879 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-12T22:50:00.886 INFO:teuthology.task.internal:Creating test directory... 2024-04-12T22:50:00.886 DEBUG:teuthology.orchestra.run.smithi144:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T22:50:00.889 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-12T22:50:00.903 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-12T22:50:00.933 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-12T22:50:00.971 INFO:teuthology.task.internal:Creating archive directory... 2024-04-12T22:50:00.971 DEBUG:teuthology.orchestra.run.smithi144:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T22:50:00.973 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-12T22:50:01.007 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-12T22:50:01.133 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-12T22:50:01.133 DEBUG:teuthology.orchestra.run.smithi144:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T22:50:01.136 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-12T22:50:01.168 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T22:50:01.181 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T22:50:01.182 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T22:50:01.196 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-12T22:50:01.198 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-12T22:50:01.205 INFO:teuthology.task.internal:Configuring sudo... 2024-04-12T22:50:01.205 DEBUG:teuthology.orchestra.run.smithi144:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T22:50:01.225 DEBUG:teuthology.orchestra.run.smithi181:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-12T22:50:01.251 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-12T22:50:01.259 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-12T22:50:01.259 DEBUG:teuthology.orchestra.run.smithi144:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T22:50:01.292 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-12T22:50:01.305 DEBUG:teuthology.orchestra.run.smithi144:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T22:50:01.383 DEBUG:teuthology.orchestra.run.smithi144:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T22:50:01.444 DEBUG:teuthology.orchestra.run.smithi144:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T22:50:01.525 DEBUG:teuthology.orchestra.run.smithi144:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T22:50:01.567 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:50:01.567 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T22:50:01.634 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T22:50:01.675 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-12T22:50:01.728 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T22:50:01.770 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-12T22:50:01.809 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-04-12T22:50:01.809 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-12T22:50:01.873 DEBUG:teuthology.orchestra.run.smithi144:> sudo service rsyslog restart 2024-04-12T22:50:01.875 DEBUG:teuthology.orchestra.run.smithi181:> sudo service rsyslog restart 2024-04-12T22:50:01.933 INFO:teuthology.orchestra.run.smithi144.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T22:50:01.957 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T22:50:02.313 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-12T22:50:02.322 INFO:teuthology.task.internal:Starting timer... 2024-04-12T22:50:02.322 INFO:teuthology.run_tasks:Running task pcp... 2024-04-12T22:50:02.358 INFO:teuthology.run_tasks:Running task selinux... 2024-04-12T22:50:02.390 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-12T22:50:02.390 DEBUG:teuthology.orchestra.run.smithi144:> sudo service auditd rotate 2024-04-12T22:50:02.452 INFO:teuthology.orchestra.run.smithi144.stdout:Rotating logs: 2024-04-12T22:50:02.453 DEBUG:teuthology.orchestra.run.smithi181:> sudo service auditd rotate 2024-04-12T22:50:02.527 INFO:teuthology.orchestra.run.smithi181.stdout:Rotating logs: 2024-04-12T22:50:02.528 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-12T22:50:02.528 DEBUG:teuthology.orchestra.run.smithi144:> /usr/sbin/getenforce 2024-04-12T22:50:02.570 INFO:teuthology.orchestra.run.smithi144.stdout:Permissive 2024-04-12T22:50:02.571 DEBUG:teuthology.orchestra.run.smithi181:> /usr/sbin/getenforce 2024-04-12T22:50:02.593 INFO:teuthology.orchestra.run.smithi181.stdout:Permissive 2024-04-12T22:50:02.593 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi144.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi181.front.sepia.ceph.com': 'permissive'} 2024-04-12T22:50:02.594 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T22:50:02.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:50:02.638 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T22:50:02.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T22:50:02.666 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-12T22:50:02.666 DEBUG:teuthology.orchestra.run.smithi144:> sudo /usr/sbin/setenforce permissive 2024-04-12T22:50:02.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo /usr/sbin/setenforce permissive 2024-04-12T22:50:02.787 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-12T22:50:02.796 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-12T22:50:02.811 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-12T22:50:02.812 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi144.front.sepia.ceph.com,smithi181.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-12T22:54:20.054 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi144.front.sepia.ceph.com'), Remote(name='ubuntu@smithi181.front.sepia.ceph.com')] 2024-04-12T22:54:20.055 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:54:20.056 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi144.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:54:20.151 DEBUG:teuthology.orchestra.run.smithi144:> true 2024-04-12T22:54:20.229 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi144.front.sepia.ceph.com' 2024-04-12T22:54:20.229 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:54:20.230 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-12T22:54:20.363 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-04-12T22:54:20.447 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-04-12T22:54:20.447 INFO:teuthology.run_tasks:Running task clock... 2024-04-12T22:54:20.460 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-12T22:54:20.461 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T22:54:20.463 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T22:54:20.465 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-12T22:54:20.466 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T22:54:20.497 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T22:54:20.516 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T22:54:20.519 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-12T22:54:20.534 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-12T22:54:20.545 INFO:teuthology.orchestra.run.smithi144.stderr:sudo: ntpd: command not found 2024-04-12T22:54:20.559 INFO:teuthology.orchestra.run.smithi144.stdout:506 Cannot talk to daemon 2024-04-12T22:54:20.562 INFO:teuthology.orchestra.run.smithi181.stderr:sudo: ntpd: command not found 2024-04-12T22:54:20.577 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T22:54:20.577 INFO:teuthology.orchestra.run.smithi181.stdout:506 Cannot talk to daemon 2024-04-12T22:54:20.592 INFO:teuthology.orchestra.run.smithi144.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T22:54:20.593 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-12T22:54:20.609 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-12T22:54:20.642 INFO:teuthology.orchestra.run.smithi144.stderr:bash: line 1: ntpq: command not found 2024-04-12T22:54:20.646 INFO:teuthology.orchestra.run.smithi144.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T22:54:20.649 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================== 2024-04-12T22:54:20.649 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.650 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.650 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.650 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.664 INFO:teuthology.orchestra.run.smithi181.stderr:bash: line 1: ntpq: command not found 2024-04-12T22:54:20.668 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T22:54:20.668 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2024-04-12T22:54:20.668 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.668 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.668 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.669 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T22:54:20.669 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-12T22:54:20.791 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b277feb2c7beb4597d8ecc39d58d5c36b81f613a'} 2024-04-12T22:54:20.791 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-12T22:54:20.792 INFO:tasks.cephadm:Cluster fsid is 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:20.792 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-12T22:54:20.792 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-12T22:54:20.792 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi144': '172.21.15.144', 'mon.smithi181': '172.21.15.181'} 2024-04-12T22:54:20.792 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-12T22:54:20.792 DEBUG:teuthology.orchestra.run.smithi144:> sudo hostname $(hostname -s) 2024-04-12T22:54:20.820 DEBUG:teuthology.orchestra.run.smithi181:> sudo hostname $(hostname -s) 2024-04-12T22:54:20.848 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-12T22:54:20.849 DEBUG:teuthology.orchestra.run.smithi144:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T22:54:21.024 INFO:teuthology.orchestra.run.smithi144.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 12 22:54 /home/ubuntu/cephtest/cephadm 2024-04-12T22:54:21.024 DEBUG:teuthology.orchestra.run.smithi181:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-12T22:54:21.189 INFO:teuthology.orchestra.run.smithi181.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 12 22:54 /home/ubuntu/cephtest/cephadm 2024-04-12T22:54:21.189 DEBUG:teuthology.orchestra.run.smithi144:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T22:54:21.205 DEBUG:teuthology.orchestra.run.smithi181:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-12T22:54:21.229 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-12T22:54:21.229 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-12T22:54:21.248 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-12T22:54:22.135 INFO:teuthology.orchestra.run.smithi181.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-12T22:54:22.170 INFO:teuthology.orchestra.run.smithi144.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-12T22:54:45.069 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2024-04-12T22:54:45.069 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-12T22:54:45.069 INFO:teuthology.orchestra.run.smithi181.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-12T22:54:45.070 INFO:teuthology.orchestra.run.smithi181.stdout: "repo_digests": [ 2024-04-12T22:54:45.070 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-12T22:54:45.070 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-12T22:54:45.070 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2024-04-12T22:54:45.070 INFO:teuthology.orchestra.run.smithi181.stdout:} 2024-04-12T22:54:45.088 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T22:54:45.088 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-12T22:54:45.089 INFO:teuthology.orchestra.run.smithi144.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-12T22:54:45.089 INFO:teuthology.orchestra.run.smithi144.stdout: "repo_digests": [ 2024-04-12T22:54:45.089 INFO:teuthology.orchestra.run.smithi144.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-12T22:54:45.089 INFO:teuthology.orchestra.run.smithi144.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-12T22:54:45.089 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2024-04-12T22:54:45.089 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T22:54:45.107 DEBUG:teuthology.orchestra.run.smithi144:> sudo mkdir -p /etc/ceph 2024-04-12T22:54:45.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo mkdir -p /etc/ceph 2024-04-12T22:54:45.202 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 777 /etc/ceph 2024-04-12T22:54:45.229 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 777 /etc/ceph 2024-04-12T22:54:45.269 INFO:tasks.cephadm:Writing seed config... 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-12T22:54:45.271 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-12T22:54:45.272 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:54:45.272 DEBUG:teuthology.orchestra.run.smithi144:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-12T22:54:45.286 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9965523c-f91f-11ee-bc8d-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-12T22:54:45.287 DEBUG:teuthology.orchestra.run.smithi144:mon.smithi144> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service 2024-04-12T22:54:45.329 INFO:tasks.cephadm:Bootstrapping... 2024-04-12T22:54:45.329 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.144 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:54:45.523 INFO:teuthology.orchestra.run.smithi144.stderr:-------------------------------------------------------------------------------- 2024-04-12T22:54:45.524 INFO:teuthology.orchestra.run.smithi144.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '9965523c-f91f-11ee-bc8d-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.144', '--skip-admin-label'] 2024-04-12T22:54:45.547 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: 4.9.4-dev 2024-04-12T22:54:45.549 INFO:teuthology.orchestra.run.smithi144.stderr:Verifying podman|docker is present... 2024-04-12T22:54:45.570 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: 4.9.4-dev 2024-04-12T22:54:45.572 INFO:teuthology.orchestra.run.smithi144.stderr:Verifying lvm2 is present... 2024-04-12T22:54:45.572 INFO:teuthology.orchestra.run.smithi144.stderr:Verifying time synchronization is in place... 2024-04-12T22:54:45.580 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T22:54:45.586 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: inactive 2024-04-12T22:54:45.592 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: enabled 2024-04-12T22:54:45.598 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: active 2024-04-12T22:54:45.599 INFO:teuthology.orchestra.run.smithi144.stderr:Unit chronyd.service is enabled and running 2024-04-12T22:54:45.599 INFO:teuthology.orchestra.run.smithi144.stderr:Repeating the final host check... 2024-04-12T22:54:45.619 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: 4.9.4-dev 2024-04-12T22:54:45.621 INFO:teuthology.orchestra.run.smithi144.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-12T22:54:45.621 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl is present 2024-04-12T22:54:45.621 INFO:teuthology.orchestra.run.smithi144.stderr:lvcreate is present 2024-04-12T22:54:45.628 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-12T22:54:45.633 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: inactive 2024-04-12T22:54:45.639 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: enabled 2024-04-12T22:54:45.646 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: active 2024-04-12T22:54:45.647 INFO:teuthology.orchestra.run.smithi144.stderr:Unit chronyd.service is enabled and running 2024-04-12T22:54:45.647 INFO:teuthology.orchestra.run.smithi144.stderr:Host looks OK 2024-04-12T22:54:45.647 INFO:teuthology.orchestra.run.smithi144.stderr:Cluster fsid: 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:45.647 INFO:teuthology.orchestra.run.smithi144.stderr:Acquiring lock 140670833847696 on /run/cephadm/9965523c-f91f-11ee-bc8d-c7b262605968.lock 2024-04-12T22:54:45.647 INFO:teuthology.orchestra.run.smithi144.stderr:Lock 140670833847696 acquired on /run/cephadm/9965523c-f91f-11ee-bc8d-c7b262605968.lock 2024-04-12T22:54:45.647 INFO:teuthology.orchestra.run.smithi144.stderr:Verifying IP 172.21.15.144 port 3300 ... 2024-04-12T22:54:45.648 INFO:teuthology.orchestra.run.smithi144.stderr:Verifying IP 172.21.15.144 port 6789 ... 2024-04-12T22:54:45.648 INFO:teuthology.orchestra.run.smithi144.stderr:Base mon IP is 172.21.15.144, final addrv is [v2:172.21.15.144:3300,v1:172.21.15.144:6789] 2024-04-12T22:54:45.650 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.144 metric 100 2024-04-12T22:54:45.650 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.144 metric 100 2024-04-12T22:54:45.652 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-12T22:54:45.653 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-12T22:54:45.653 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1774sec hoplimit 64 pref medium 2024-04-12T22:54:45.655 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-12T22:54:45.655 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-12T22:54:45.655 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-12T22:54:45.656 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-12T22:54:45.656 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe88:7f5f/64 scope link 2024-04-12T22:54:45.656 INFO:teuthology.orchestra.run.smithi144.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-12T22:54:45.656 INFO:teuthology.orchestra.run.smithi144.stderr:Mon IP `172.21.15.144` is in CIDR network `172.21.0.0/20` 2024-04-12T22:54:45.656 INFO:teuthology.orchestra.run.smithi144.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-12T22:54:45.657 INFO:teuthology.orchestra.run.smithi144.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-12T22:54:45.757 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-12T22:54:46.120 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Getting image source signatures 2024-04-12T22:54:46.121 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-12T22:54:46.121 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-12T22:54:46.121 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-12T22:54:46.121 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-12T22:54:46.121 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-12T22:54:46.121 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-12T22:54:46.276 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: Writing manifest to image destination 2024-04-12T22:54:46.429 INFO:teuthology.orchestra.run.smithi144.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-12T22:54:46.851 INFO:teuthology.orchestra.run.smithi144.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-12T22:54:47.389 INFO:teuthology.orchestra.run.smithi144.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-12T22:54:47.389 INFO:teuthology.orchestra.run.smithi144.stderr:Extracting ceph user uid/gid from container image... 2024-04-12T22:54:47.813 INFO:teuthology.orchestra.run.smithi144.stderr:stat: 167 167 2024-04-12T22:54:48.334 INFO:teuthology.orchestra.run.smithi144.stderr:Creating initial keys... 2024-04-12T22:54:48.741 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph-authtool: AQC4uxlm40ZBJxAA7l9my8VZcwXLTy/zygYcog== 2024-04-12T22:54:49.663 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph-authtool: AQC5uxlm1sSSIxAAXQAzeYbTj+tl8DXhQ+Y0Tw== 2024-04-12T22:54:50.486 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph-authtool: AQC6uxlm6d2AGRAAjzHFlLFS7ZdAv9Gif/ngWQ== 2024-04-12T22:54:51.034 INFO:teuthology.orchestra.run.smithi144.stderr:Creating initial monmap... 2024-04-12T22:54:51.400 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T22:54:51.400 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-12T22:54:51.400 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:51.400 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T22:54:51.873 INFO:teuthology.orchestra.run.smithi144.stderr:monmaptool for smithi144 [v2:172.21.15.144:3300,v1:172.21.15.144:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-12T22:54:51.873 INFO:teuthology.orchestra.run.smithi144.stderr:setting min_mon_release = octopus 2024-04-12T22:54:51.873 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/monmaptool: set fsid to 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:51.873 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-12T22:54:51.873 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:54:51.873 INFO:teuthology.orchestra.run.smithi144.stderr:Creating mon... 2024-04-12T22:54:52.853 INFO:teuthology.orchestra.run.smithi144.stderr:create mon.smithi144 on 2024-04-12T22:54:53.060 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-12T22:54:53.209 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9965523c-f91f-11ee-bc8d-c7b262605968.target → /etc/systemd/system/ceph-9965523c-f91f-11ee-bc8d-c7b262605968.target. 2024-04-12T22:54:53.209 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9965523c-f91f-11ee-bc8d-c7b262605968.target → /etc/systemd/system/ceph-9965523c-f91f-11ee-bc8d-c7b262605968.target. 2024-04-12T22:54:53.505 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Failed to reset failed state of unit ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service: Unit ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service not loaded. 2024-04-12T22:54:53.512 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9965523c-f91f-11ee-bc8d-c7b262605968.target.wants/ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service → /etc/systemd/system/ceph-9965523c-f91f-11ee-bc8d-c7b262605968@.service. 2024-04-12T22:54:54.139 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:54 smithi144 podman[24449]: 2024-04-12 22:54:54.108871302 +0000 UTC m=+0.316098128 container start be17038a0f91c0b5c2226629d2dd02b967f9b796de562e2b2c80b69110c4861e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container) 2024-04-12T22:54:54.208 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: disabled 2024-04-12T22:54:54.214 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: inactive 2024-04-12T22:54:54.215 INFO:teuthology.orchestra.run.smithi144.stderr:firewalld.service is not enabled 2024-04-12T22:54:54.215 INFO:teuthology.orchestra.run.smithi144.stderr:Not possible to enable service . firewalld.service is not available 2024-04-12T22:54:54.215 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for mon to start... 2024-04-12T22:54:54.215 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for mon... 2024-04-12T22:54:54.411 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:54 smithi144 ceph-mon[24468]: mkfs 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:54.411 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:54 smithi144 ceph-mon[24468]: mon.smithi144 is new leader, mons smithi144 in quorum (ranks 0) 2024-04-12T22:54:54.411 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:54 smithi144 bash[24449]: be17038a0f91c0b5c2226629d2dd02b967f9b796de562e2b2c80b69110c4861e 2024-04-12T22:54:54.411 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:54 smithi144 systemd[1]: Started Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: cluster: 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: id: 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: services: 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi144 (age 0.592377s) 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-12T22:54:54.728 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: data: 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: pgs: 2024-04-12T22:54:54.729 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:54:55.263 INFO:teuthology.orchestra.run.smithi144.stderr:mon is available 2024-04-12T22:54:55.263 INFO:teuthology.orchestra.run.smithi144.stderr:Assimilating anything we can from ceph.conf... 2024-04-12T22:54:55.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:55 smithi144 ceph-mon[24468]: mon.smithi144 is new leader, mons smithi144 in quorum (ranks 0) 2024-04-12T22:54:55.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:55 smithi144 ceph-mon[24468]: monmap e1: 1 mons at {smithi144=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} 2024-04-12T22:54:55.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:55 smithi144 ceph-mon[24468]: fsmap 2024-04-12T22:54:55.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:55 smithi144 ceph-mon[24468]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T22:54:55.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:55 smithi144 ceph-mon[24468]: mgrmap e1: no daemons active 2024-04-12T22:54:55.467 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:55 smithi144 ceph-mon[24468]: from='client.? 172.21.15.144:0/1500762793' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-12T22:54:55.748 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:54:55.748 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: [global] 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: fsid = 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.144:3300,v1:172.21.15.144:6789] 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: [mgr] 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-12T22:54:55.749 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:54:55.750 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: [osd] 2024-04-12T22:54:55.750 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-12T22:54:55.750 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-12T22:54:56.335 INFO:teuthology.orchestra.run.smithi144.stderr:Generating new minimal ceph.conf... 2024-04-12T22:54:56.450 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:56 smithi144 ceph-mon[24468]: from='client.? 172.21.15.144:0/1517401604' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T22:54:56.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:56 smithi144 ceph-mon[24468]: from='client.? 172.21.15.144:0/1517401604' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-12T22:54:57.333 INFO:teuthology.orchestra.run.smithi144.stderr:Restarting the monitor... 2024-04-12T22:54:57.492 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 ceph-mon[24468]: from='client.? 172.21.15.144:0/617251365' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:54:57.493 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 systemd[1]: Stopping Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T22:54:57.493 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 bash[24687]: Error: no container with name or ID "ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon.smithi144" found: no such container 2024-04-12T22:54:57.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[24464]: 2024-04-12T22:54:57.502+0000 7f5ca3a0d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi144 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T22:54:57.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[24464]: 2024-04-12T22:54:57.502+0000 7f5ca3a0d700 -1 mon.smithi144@0(leader) e1 *** Got Signal Terminated *** 2024-04-12T22:54:57.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 podman[24700]: 2024-04-12 22:54:57.666097453 +0000 UTC m=+0.269745515 container died be17038a0f91c0b5c2226629d2dd02b967f9b796de562e2b2c80b69110c4861e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, release=754, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., io.openshift.expose-services=, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870) 2024-04-12T22:54:58.184 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 podman[24700]: 2024-04-12 22:54:57.985839984 +0000 UTC m=+0.589488046 container cleanup be17038a0f91c0b5c2226629d2dd02b967f9b796de562e2b2c80b69110c4861e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, distribution-scope=public) 2024-04-12T22:54:58.184 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:57 smithi144 bash[24700]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144 2024-04-12T22:54:58.184 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 bash[24728]: Error: no container with name or ID "ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon.smithi144" found: no such container 2024-04-12T22:54:58.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 podman[24715]: 2024-04-12 22:54:58.194834555 +0000 UTC m=+0.525046586 container remove be17038a0f91c0b5c2226629d2dd02b967f9b796de562e2b2c80b69110c4861e (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , name=centos-stream, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T22:54:58.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service: Deactivated successfully. 2024-04-12T22:54:58.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 systemd[1]: Stopped Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T22:54:58.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 systemd[1]: Starting Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T22:54:58.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 podman[24796]: 2024-04-12 22:54:58.412222678 +0000 UTC m=+0.102490002 container create cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, io.buildah.version=1.19.8, GIT_CLEAN=True, name=centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754) 2024-04-12T22:54:58.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 podman[24796]: 2024-04-12 22:54:58.328085611 +0000 UTC m=+0.018352953 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-12T22:54:58.715 INFO:teuthology.orchestra.run.smithi144.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-12T22:54:58.840 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 podman[24796]: 2024-04-12 22:54:58.647934187 +0000 UTC m=+0.338201511 container init cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.buildah.version=1.19.8, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=) 2024-04-12T22:54:58.841 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 podman[24796]: 2024-04-12 22:54:58.650998161 +0000 UTC m=+0.341265489 container start cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , architecture=x86_64, RELEASE=HEAD, version=8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-12T22:54:58.841 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T22:54:58.841 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-12T22:54:58.841 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: pidfile_write: ignore empty --pid-file 2024-04-12T22:54:58.841 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: load: jerasure load: lrc 2024-04-12T22:54:58.841 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: RocksDB version: 6.15.5 2024-04-12T22:54:58.844 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-12T22:54:58.845 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Compile date Apr 18 2022 2024-04-12T22:54:58.845 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: DB SUMMARY 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: DB Session ID: DQEV8C82IL2ERSAGOG9Y 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: CURRENT file: CURRENT 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi144/store.db dir, Total Num: 1, files: 000008.sst 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi144/store.db: 000010.log size: 82739 ; 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.error_if_exists: 0 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.create_if_missing: 0 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.paranoid_checks: 1 2024-04-12T22:54:58.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T22:54:58.847 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.env: 0x562648cc6860 2024-04-12T22:54:58.847 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.fs: Posix File System 2024-04-12T22:54:58.847 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.info_log: 0x5626498adfa0 2024-04-12T22:54:58.847 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T22:54:58.847 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.statistics: (nil) 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.use_fsync: 0 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_log_file_size: 0 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T22:54:58.848 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.allow_fallocate: 1 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.use_direct_reads: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.db_log_dir: 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi144/store.db 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T22:54:58.849 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.write_buffer_manager: 0x56264999e2d0 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T22:54:58.850 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.unordered_write: 0 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.row_cache: None 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.wal_filter: None 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.preserve_deletes: 0 2024-04-12T22:54:58.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.two_write_queues: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.atomic_flush: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.log_readahead_size: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T22:54:58.852 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_background_jobs: 2 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_background_compactions: -1 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_subcompactions: 1 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_open_files: -1 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T22:54:58.853 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_background_flushes: -1 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Compression algorithms supported: 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kZSTD supported: 0 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kXpressCompression supported: 0 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T22:54:58.854 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kLZ4Compression supported: 1 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kBZip2Compression supported: 0 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kZlibCompression supported: 1 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: kSnappyCompression supported: 1 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi144/store.db/MANIFEST-000009 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.merge_operator: 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_filter: None 2024-04-12T22:54:58.855 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562649879d00) 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: cache_index_and_filter_blocks: 1 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: pin_top_level_index_and_filter: 1 2024-04-12T22:54:58.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_type: 0 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_block_index_type: 0 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_shortening: 1 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: hash_index_allow_collision: 1 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: checksum: 1 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: no_block_cache: 0 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache: 0x5626498e4170 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_name: BinnedLRUCache 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_options: 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: capacity : 536870912 2024-04-12T22:54:58.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: num_shard_bits : 4 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: strict_capacity_limit : 0 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: high_pri_pool_ratio: 0.000 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_compressed: (nil) 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: persistent_cache: (nil) 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_size: 4096 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_size_deviation: 10 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_restart_interval: 16 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_block_restart_interval: 1 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: metadata_block_size: 4096 2024-04-12T22:54:58.858 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: partition_filters: 0 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: use_delta_encoding: 1 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: whole_key_filtering: 1 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: verify_compression: 0 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: read_amp_bytes_per_bit: 0 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: format_version: 4 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: enable_index_compression: 1 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_align: 0 2024-04-12T22:54:58.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression: NoCompression 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.num_levels: 7 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T22:54:58.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T22:54:58.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T22:54:58.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.arena_block_size: 4194304 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-12T22:54:58.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T22:54:58.864 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T22:54:58.864 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T22:54:58.864 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T22:54:58.864 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T22:54:58.864 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.table_properties_collectors: 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.inplace_update_support: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.bloom_locality: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.max_successive_merges: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T22:54:58.865 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.ttl: 2592000 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.enable_blob_files: false 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.min_blob_size: 0 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi144/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712962498678217, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-12T22:54:58.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712962498678923, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79421, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77685, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71994, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712962498, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "205fa14b-42bf-4c09-8673-95bf641a1ec9", "db_session_id": "DQEV8C82IL2ERSAGOG9Y"}} 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712962498679267, "job": 1, "event": "recovery_finished"} 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi144/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5626498caa80 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: DB pointer 0x5626498da000 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** DB Stats ** 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T22:54:58.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:54:58.868 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative compaction: 0.00 GB write, 25.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval compaction: 0.00 GB write, 25.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:54:58.869 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: L0 2/0 79.36 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Sum 2/0 79.36 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.5 0.00 0.00 1 0.000 0 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative compaction: 0.00 GB write, 25.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:54:58.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: starting mon.smithi144 rank 0 at public addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] at bind addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi144 fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???) e1 preinit fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).mds e1 new map 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).mds e1 print_map 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: e1 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: legacy client fscid: -1 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: No filesystems configured 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T22:54:58.871 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 bash[24796]: cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 systemd[1]: Started Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mon.smithi144 is new leader, mons smithi144 in quorum (ranks 0) 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: monmap e1: 1 mons at {smithi144=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0]} 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: fsmap 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: osdmap e1: 0 total, 0 up, 0 in 2024-04-12T22:54:58.872 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:54:58 smithi144 ceph-mon[24815]: mgrmap e1: no daemons active 2024-04-12T22:54:59.772 INFO:teuthology.orchestra.run.smithi144.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-12T22:54:59.772 INFO:teuthology.orchestra.run.smithi144.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:54:59.772 INFO:teuthology.orchestra.run.smithi144.stderr:Creating mgr... 2024-04-12T22:54:59.772 INFO:teuthology.orchestra.run.smithi144.stderr:Verifying port 9283 ... 2024-04-12T22:54:59.939 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Failed to reset failed state of unit ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mgr.smithi144.rlueky.service: Unit ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mgr.smithi144.rlueky.service not loaded. 2024-04-12T22:54:59.946 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9965523c-f91f-11ee-bc8d-c7b262605968.target.wants/ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mgr.smithi144.rlueky.service → /etc/systemd/system/ceph-9965523c-f91f-11ee-bc8d-c7b262605968@.service. 2024-04-12T22:55:00.614 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:00 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1092889809' entity='client.admin' 2024-04-12T22:55:00.627 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: disabled 2024-04-12T22:55:00.635 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: inactive 2024-04-12T22:55:00.635 INFO:teuthology.orchestra.run.smithi144.stderr:firewalld.service is not enabled 2024-04-12T22:55:00.635 INFO:teuthology.orchestra.run.smithi144.stderr:Not possible to enable service . firewalld.service is not available 2024-04-12T22:55:00.643 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: disabled 2024-04-12T22:55:00.650 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: inactive 2024-04-12T22:55:00.651 INFO:teuthology.orchestra.run.smithi144.stderr:firewalld.service is not enabled 2024-04-12T22:55:00.651 INFO:teuthology.orchestra.run.smithi144.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-12T22:55:00.651 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for mgr to start... 2024-04-12T22:55:00.652 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for mgr... 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsid": "9965523c-f91f-11ee-bc8d-c7b262605968", 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "health": { 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T22:55:01.208 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 0 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "smithi144" 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T22:55:01.209 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T22:55:01.210 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T22:55:01.213 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "available": false, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "iostat", 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "nfs", 2024-04-12T22:55:01.214 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "restful" 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modified": "2024-04-12T22:54:54.146235+0000", 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T22:55:01.215 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:01.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:01 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/356810627' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T22:55:01.639 INFO:teuthology.orchestra.run.smithi144.stderr:mgr not available, waiting (1/15)... 2024-04-12T22:55:04.108 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:04.108 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:04.108 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsid": "9965523c-f91f-11ee-bc8d-c7b262605968", 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "health": { 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 0 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "smithi144" 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T22:55:04.109 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T22:55:04.110 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T22:55:04.114 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T22:55:04.115 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T22:55:04.115 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.115 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T22:55:04.115 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "available": false, 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "iostat", 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "nfs", 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "restful" 2024-04-12T22:55:04.116 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modified": "2024-04-12T22:54:54.146235+0000", 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T22:55:04.117 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:04.646 INFO:teuthology.orchestra.run.smithi144.stderr:mgr not available, waiting (2/15)... 2024-04-12T22:55:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:04 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3051581458' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T22:55:06.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: Activating manager daemon smithi144.rlueky 2024-04-12T22:55:06.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: mgrmap e2: smithi144.rlueky(active, starting, since 0.00285755s) 2024-04-12T22:55:06.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T22:55:06.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: Manager daemon smithi144.rlueky is now available 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:06.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:06 smithi144 ceph-mon[24815]: from='mgr.14100 172.21.15.144:0/3437244860' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsid": "9965523c-f91f-11ee-bc8d-c7b262605968", 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "health": { 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T22:55:07.135 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 0 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "smithi144" 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T22:55:07.136 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.137 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "available": false, 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "iostat", 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "nfs", 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "restful" 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.138 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T22:55:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modified": "2024-04-12T22:54:54.146235+0000", 2024-04-12T22:55:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T22:55:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:07.650 INFO:teuthology.orchestra.run.smithi144.stderr:mgr not available, waiting (3/15)... 2024-04-12T22:55:07.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:07 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2566660200' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T22:55:07.771 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:07 smithi144 ceph-mon[24815]: mgrmap e3: smithi144.rlueky(active, since 1.00594s) 2024-04-12T22:55:09.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:09 smithi144 ceph-mon[24815]: mgrmap e4: smithi144.rlueky(active, since 2s) 2024-04-12T22:55:13.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:13 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3539505244' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsid": "9965523c-f91f-11ee-bc8d-c7b262605968", 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "health": { 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "checks": {}, 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mutes": [] 2024-04-12T22:55:13.978 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum": [ 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 0 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "smithi144" 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "quorum_age": 14, 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "monmap": { 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osdmap": { 2024-04-12T22:55:13.979 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:13.980 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-12T22:55:13.980 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-12T22:55:13.980 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-12T22:55:13.984 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-12T22:55:13.984 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgmap": { 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "fsmap": { 2024-04-12T22:55:13.985 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "available": true, 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modules": [ 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "iostat", 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "nfs", 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "restful" 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ], 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "servicemap": { 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-12T22:55:13.986 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "modified": "2024-04-12T22:54:54.146235+0000", 2024-04-12T22:55:13.987 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "services": {} 2024-04-12T22:55:13.987 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: }, 2024-04-12T22:55:13.987 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-12T22:55:13.987 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:19.118 INFO:teuthology.orchestra.run.smithi144.stderr:mgr is available 2024-04-12T22:55:21.676 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: [global] 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: fsid = 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: [mgr] 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: [osd] 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-12T22:55:21.677 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-12T22:55:22.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:21 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2724104708' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-12T22:55:22.159 INFO:teuthology.orchestra.run.smithi144.stderr:Enabling cephadm module... 2024-04-12T22:55:23.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:22 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/77957692' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-12T22:55:24.041 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:24.042 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-12T22:55:24.042 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "available": true, 2024-04-12T22:55:24.042 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "active_name": "smithi144.rlueky", 2024-04-12T22:55:24.042 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-12T22:55:24.042 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:24.152 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:23 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/77957692' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-12T22:55:24.152 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:23 smithi144 ceph-mon[24815]: mgrmap e5: smithi144.rlueky(active, since 16s) 2024-04-12T22:55:24.613 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for the mgr to restart... 2024-04-12T22:55:24.613 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for mgr epoch 5... 2024-04-12T22:55:25.013 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:24 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1470498563' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T22:55:30.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:29 smithi144 ceph-mon[24815]: Active manager daemon smithi144.rlueky restarted 2024-04-12T22:55:30.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:29 smithi144 ceph-mon[24815]: Activating manager daemon smithi144.rlueky 2024-04-12T22:55:30.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:29 smithi144 ceph-mon[24815]: osdmap e2: 0 total, 0 up, 0 in 2024-04-12T22:55:30.693 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:30.693 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-12T22:55:30.693 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "initialized": true 2024-04-12T22:55:30.694 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:30.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: mgrmap e6: smithi144.rlueky(active, starting, since 0.972919s) 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: Manager daemon smithi144.rlueky is now available 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:55:30.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:55:30.777 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:30.777 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:30 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:31.294 INFO:teuthology.orchestra.run.smithi144.stderr:mgr epoch 5 is available 2024-04-12T22:55:31.295 INFO:teuthology.orchestra.run.smithi144.stderr:Setting orchestrator backend to cephadm... 2024-04-12T22:55:31.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:31 smithi144 ceph-mon[24815]: [12/Apr/2024:22:55:30] ENGINE Bus STARTING 2024-04-12T22:55:31.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:31 smithi144 ceph-mon[24815]: [12/Apr/2024:22:55:30] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T22:55:31.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:31 smithi144 ceph-mon[24815]: [12/Apr/2024:22:55:30] ENGINE Bus STARTED 2024-04-12T22:55:31.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:31 smithi144 ceph-mon[24815]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T22:55:31.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:31 smithi144 ceph-mon[24815]: mgrmap e7: smithi144.rlueky(active, since 1.977s) 2024-04-12T22:55:31.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:31 smithi144 ceph-mon[24815]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T22:55:33.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:32 smithi144 ceph-mon[24815]: mgrmap e8: smithi144.rlueky(active, since 2s) 2024-04-12T22:55:33.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:32 smithi144 ceph-mon[24815]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:33.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:32 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:33.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:32 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:33.057 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: value unchanged 2024-04-12T22:55:33.481 INFO:teuthology.orchestra.run.smithi144.stderr:Generating ssh key... 2024-04-12T22:55:34.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:34 smithi144 ceph-mon[24815]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:34.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:34 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:34.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:34 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:35.357 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/463L85iqfkkMSMpmfpQJimbHiwHXUYlrUnikY/EkguUDuZZnMELBXFAjWkJ6kEr+SOhs8X7V3n7rhyhc/qmKObOxRKHXqeMYF+0tH3rgNxi1PkZyJcv7LCsrqdtA9xlRlU4Nj4ztuVscW8p0ir4dRNN2Fp/+PoYYjLs+zkqEyHX50NRxTLdykdpsMWQy943XZmlzaEaNo8GYEmYGYcuEQMNksSfjhgqU6Sz4k15A/4ChJ16x0vVEXowue83ULdAc9mYVaA0PRxpHZn9RhZnr8/N8dAniYScxTI0qbgCvKw1pR9In/qT4aWjM9AkVB++CbAN6sBNZ5FtdZt7S+u/kVq8/4F3/uFLYHwJe2bOoCNlr8qFs98pdee+825BPcEEB1Zn1Da/+YGgpSlbwZfim1S0ML16bmbyxDDvSb+/T5DqwiAvJjSxUHRk31pojrnIGVfR1qCpdY15YwNVtRa7RekvdnIWSiMQH5i39VPHWSGlQtMEAIEi0Uomyjl9v+Kc= ceph-9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:55:35.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:35 smithi144 ceph-mon[24815]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:35.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:35 smithi144 ceph-mon[24815]: Generating ssh key... 2024-04-12T22:55:35.919 INFO:teuthology.orchestra.run.smithi144.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-12T22:55:35.919 INFO:teuthology.orchestra.run.smithi144.stderr:Adding key to root@localhost authorized_keys... 2024-04-12T22:55:35.919 INFO:teuthology.orchestra.run.smithi144.stderr:Adding host smithi144... 2024-04-12T22:55:36.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:36 smithi144 ceph-mon[24815]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:37.489 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Added host 'smithi144' with addr '172.21.15.144' 2024-04-12T22:55:37.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:37 smithi144 ceph-mon[24815]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi144", "addr": "172.21.15.144", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:37.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:37 smithi144 ceph-mon[24815]: Deploying cephadm binary to smithi144 2024-04-12T22:55:37.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:37 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:37.777 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:37 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:38.108 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying mon service with default placement... 2024-04-12T22:55:38.734 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:38 smithi144 ceph-mon[24815]: Added host smithi144 2024-04-12T22:55:38.735 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-12T22:55:39.994 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:39 smithi144 ceph-mon[24815]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:39.995 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:39 smithi144 ceph-mon[24815]: Saving service mon spec with placement count:5 2024-04-12T22:55:39.995 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:39 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:40.265 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying mgr service with default placement... 2024-04-12T22:55:40.920 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-12T22:55:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:40 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:40 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:40 smithi144 ceph-mon[24815]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:40 smithi144 ceph-mon[24815]: Saving service mgr spec with placement count:2 2024-04-12T22:55:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:40 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:41.916 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying crash service with default placement... 2024-04-12T22:55:42.560 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-12T22:55:42.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:42 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:42.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:42 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:43.114 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying prometheus service with default placement... 2024-04-12T22:55:43.969 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:43 smithi144 ceph-mon[24815]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:43.969 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:43 smithi144 ceph-mon[24815]: Saving service crash spec with placement * 2024-04-12T22:55:43.969 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:43 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:44.557 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-12T22:55:45.151 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying grafana service with default placement... 2024-04-12T22:55:45.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:45 smithi144 ceph-mon[24815]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:45.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:45 smithi144 ceph-mon[24815]: Saving service prometheus spec with placement count:1 2024-04-12T22:55:45.768 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:45 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:45.768 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-12T22:55:46.300 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying node-exporter service with default placement... 2024-04-12T22:55:46.961 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:46 smithi144 ceph-mon[24815]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:46.961 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:46 smithi144 ceph-mon[24815]: Saving service grafana spec with placement count:1 2024-04-12T22:55:46.961 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:46 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:46.962 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-12T22:55:47.556 INFO:teuthology.orchestra.run.smithi144.stderr:Deploying alertmanager service with default placement... 2024-04-12T22:55:48.235 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:47 smithi144 ceph-mon[24815]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:48.235 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:47 smithi144 ceph-mon[24815]: Saving service node-exporter spec with placement * 2024-04-12T22:55:48.236 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:47 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:48.236 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-12T22:55:49.415 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:49 smithi144 ceph-mon[24815]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:55:49.415 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:49 smithi144 ceph-mon[24815]: Saving service alertmanager spec with placement count:1 2024-04-12T22:55:49.415 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:49 smithi144 ceph-mon[24815]: from='mgr.14120 172.21.15.144:0/861718408' entity='mgr.smithi144.rlueky' 2024-04-12T22:55:50.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:50 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2899357247' entity='client.admin' 2024-04-12T22:55:50.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:50 smithi144 ceph-mon[24815]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:55:51.218 INFO:teuthology.orchestra.run.smithi144.stderr:Enabling the dashboard module... 2024-04-12T22:55:52.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:51 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/845232637' entity='client.admin' 2024-04-12T22:55:52.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:52 smithi144 ceph-mon[24815]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:55:52.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:52 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3099112401' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-12T22:55:53.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:53 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3099112401' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-12T22:55:53.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:53 smithi144 ceph-mon[24815]: mgrmap e9: smithi144.rlueky(active, since 23s) 2024-04-12T22:55:53.816 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:53.816 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-12T22:55:53.816 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "available": true, 2024-04-12T22:55:53.816 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "active_name": "smithi144.rlueky", 2024-04-12T22:55:53.816 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-12T22:55:53.816 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:54.455 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for the mgr to restart... 2024-04-12T22:55:54.455 INFO:teuthology.orchestra.run.smithi144.stderr:Waiting for mgr epoch 9... 2024-04-12T22:55:54.915 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:54 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2643933908' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-12T22:55:58.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:58 smithi144 ceph-mon[24815]: Active manager daemon smithi144.rlueky restarted 2024-04-12T22:55:58.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:58 smithi144 ceph-mon[24815]: Activating manager daemon smithi144.rlueky 2024-04-12T22:55:58.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:58 smithi144 ceph-mon[24815]: osdmap e3: 0 total, 0 up, 0 in 2024-04-12T22:55:59.529 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: { 2024-04-12T22:55:59.529 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-12T22:55:59.529 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: "initialized": true 2024-04-12T22:55:59.529 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: } 2024-04-12T22:55:59.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: mgrmap e10: smithi144.rlueky(active, starting, since 0.0552834s) 2024-04-12T22:55:59.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:55:59.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T22:55:59.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T22:55:59.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: Manager daemon smithi144.rlueky is now available 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:55:59.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:55:59 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:00.166 INFO:teuthology.orchestra.run.smithi144.stderr:mgr epoch 9 is available 2024-04-12T22:56:00.166 INFO:teuthology.orchestra.run.smithi144.stderr:Generating a dashboard self-signed certificate... 2024-04-12T22:56:00.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: [12/Apr/2024:22:55:59] ENGINE Bus STARTING 2024-04-12T22:56:00.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: [12/Apr/2024:22:55:59] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T22:56:00.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: [12/Apr/2024:22:55:59] ENGINE Bus STARTED 2024-04-12T22:56:00.696 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-12T22:56:00.696 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: mgrmap e11: smithi144.rlueky(active, since 1.06393s) 2024-04-12T22:56:00.696 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-12T22:56:00.696 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:00 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:00.934 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-12T22:56:01.514 INFO:teuthology.orchestra.run.smithi144.stderr:Creating initial admin user... 2024-04-12T22:56:01.924 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:01 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:01.924 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:01 smithi144 ceph-mon[24815]: mgrmap e12: smithi144.rlueky(active, since 2s) 2024-04-12T22:56:01.924 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:01 smithi144 ceph-mon[24815]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:56:01.924 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:01 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:01.924 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:01 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:02.358 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$LnBP6tqkXiqXAb3MzebOFez421G.mIa1D3NYSZ7m1jRBXZEIKD9la", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712962562, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-12T22:56:03.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:03 smithi144 ceph-mon[24815]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:56:03.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:03 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:03.820 INFO:teuthology.orchestra.run.smithi144.stderr:Fetching dashboard port number... 2024-04-12T22:56:04.485 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: 8443 2024-04-12T22:56:04.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:04 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2637337969' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-12T22:56:05.153 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: disabled 2024-04-12T22:56:05.160 INFO:teuthology.orchestra.run.smithi144.stderr:systemctl: inactive 2024-04-12T22:56:05.161 INFO:teuthology.orchestra.run.smithi144.stderr:firewalld.service is not enabled 2024-04-12T22:56:05.161 INFO:teuthology.orchestra.run.smithi144.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-12T22:56:05.163 INFO:teuthology.orchestra.run.smithi144.stderr:Ceph Dashboard is now available at: 2024-04-12T22:56:05.163 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:05.163 INFO:teuthology.orchestra.run.smithi144.stderr: URL: https://smithi144.front.sepia.ceph.com:8443/ 2024-04-12T22:56:05.163 INFO:teuthology.orchestra.run.smithi144.stderr: User: admin 2024-04-12T22:56:05.163 INFO:teuthology.orchestra.run.smithi144.stderr: Password: ebyefwaz10 2024-04-12T22:56:05.163 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:05.164 INFO:teuthology.orchestra.run.smithi144.stderr:Enabling autotune for osd_memory_target 2024-04-12T22:56:05.814 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:05 smithi144 ceph-mon[24815]: mgrmap e13: smithi144.rlueky(active, since 6s) 2024-04-12T22:56:07.139 INFO:teuthology.orchestra.run.smithi144.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-12T22:56:07.790 INFO:teuthology.orchestra.run.smithi144.stderr:You can access the Ceph CLI with: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: ceph telemetry on 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr:For more information see: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T22:56:07.791 INFO:teuthology.orchestra.run.smithi144.stderr:Bootstrap complete. 2024-04-12T22:56:07.832 INFO:tasks.cephadm:Fetching config... 2024-04-12T22:56:07.832 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:56:07.832 DEBUG:teuthology.orchestra.run.smithi144:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-12T22:56:07.848 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-12T22:56:07.848 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:56:07.848 DEBUG:teuthology.orchestra.run.smithi144:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-12T22:56:07.904 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-12T22:56:07.904 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:56:07.904 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/keyring of=/dev/stdout 2024-04-12T22:56:07.970 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-12T22:56:07.971 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:56:07.971 DEBUG:teuthology.orchestra.run.smithi144:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-12T22:56:08.026 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-12T22:56:08.026 DEBUG:teuthology.orchestra.run.smithi144:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/463L85iqfkkMSMpmfpQJimbHiwHXUYlrUnikY/EkguUDuZZnMELBXFAjWkJ6kEr+SOhs8X7V3n7rhyhc/qmKObOxRKHXqeMYF+0tH3rgNxi1PkZyJcv7LCsrqdtA9xlRlU4Nj4ztuVscW8p0ir4dRNN2Fp/+PoYYjLs+zkqEyHX50NRxTLdykdpsMWQy943XZmlzaEaNo8GYEmYGYcuEQMNksSfjhgqU6Sz4k15A/4ChJ16x0vVEXowue83ULdAc9mYVaA0PRxpHZn9RhZnr8/N8dAniYScxTI0qbgCvKw1pR9In/qT4aWjM9AkVB++CbAN6sBNZ5FtdZt7S+u/kVq8/4F3/uFLYHwJe2bOoCNlr8qFs98pdee+825BPcEEB1Zn1Da/+YGgpSlbwZfim1S0ML16bmbyxDDvSb+/T5DqwiAvJjSxUHRk31pojrnIGVfR1qCpdY15YwNVtRa7RekvdnIWSiMQH5i39VPHWSGlQtMEAIEi0Uomyjl9v+Kc= ceph-9965523c-f91f-11ee-bc8d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T22:56:08.128 INFO:teuthology.orchestra.run.smithi144.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/463L85iqfkkMSMpmfpQJimbHiwHXUYlrUnikY/EkguUDuZZnMELBXFAjWkJ6kEr+SOhs8X7V3n7rhyhc/qmKObOxRKHXqeMYF+0tH3rgNxi1PkZyJcv7LCsrqdtA9xlRlU4Nj4ztuVscW8p0ir4dRNN2Fp/+PoYYjLs+zkqEyHX50NRxTLdykdpsMWQy943XZmlzaEaNo8GYEmYGYcuEQMNksSfjhgqU6Sz4k15A/4ChJ16x0vVEXowue83ULdAc9mYVaA0PRxpHZn9RhZnr8/N8dAniYScxTI0qbgCvKw1pR9In/qT4aWjM9AkVB++CbAN6sBNZ5FtdZt7S+u/kVq8/4F3/uFLYHwJe2bOoCNlr8qFs98pdee+825BPcEEB1Zn1Da/+YGgpSlbwZfim1S0ML16bmbyxDDvSb+/T5DqwiAvJjSxUHRk31pojrnIGVfR1qCpdY15YwNVtRa7RekvdnIWSiMQH5i39VPHWSGlQtMEAIEi0Uomyjl9v+Kc= ceph-9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:56:08.144 DEBUG:teuthology.orchestra.run.smithi181:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/463L85iqfkkMSMpmfpQJimbHiwHXUYlrUnikY/EkguUDuZZnMELBXFAjWkJ6kEr+SOhs8X7V3n7rhyhc/qmKObOxRKHXqeMYF+0tH3rgNxi1PkZyJcv7LCsrqdtA9xlRlU4Nj4ztuVscW8p0ir4dRNN2Fp/+PoYYjLs+zkqEyHX50NRxTLdykdpsMWQy943XZmlzaEaNo8GYEmYGYcuEQMNksSfjhgqU6Sz4k15A/4ChJ16x0vVEXowue83ULdAc9mYVaA0PRxpHZn9RhZnr8/N8dAniYScxTI0qbgCvKw1pR9In/qT4aWjM9AkVB++CbAN6sBNZ5FtdZt7S+u/kVq8/4F3/uFLYHwJe2bOoCNlr8qFs98pdee+825BPcEEB1Zn1Da/+YGgpSlbwZfim1S0ML16bmbyxDDvSb+/T5DqwiAvJjSxUHRk31pojrnIGVfR1qCpdY15YwNVtRa7RekvdnIWSiMQH5i39VPHWSGlQtMEAIEi0Uomyjl9v+Kc= ceph-9965523c-f91f-11ee-bc8d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-12T22:56:08.190 INFO:teuthology.orchestra.run.smithi181.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/463L85iqfkkMSMpmfpQJimbHiwHXUYlrUnikY/EkguUDuZZnMELBXFAjWkJ6kEr+SOhs8X7V3n7rhyhc/qmKObOxRKHXqeMYF+0tH3rgNxi1PkZyJcv7LCsrqdtA9xlRlU4Nj4ztuVscW8p0ir4dRNN2Fp/+PoYYjLs+zkqEyHX50NRxTLdykdpsMWQy943XZmlzaEaNo8GYEmYGYcuEQMNksSfjhgqU6Sz4k15A/4ChJ16x0vVEXowue83ULdAc9mYVaA0PRxpHZn9RhZnr8/N8dAniYScxTI0qbgCvKw1pR9In/qT4aWjM9AkVB++CbAN6sBNZ5FtdZt7S+u/kVq8/4F3/uFLYHwJe2bOoCNlr8qFs98pdee+825BPcEEB1Zn1Da/+YGgpSlbwZfim1S0ML16bmbyxDDvSb+/T5DqwiAvJjSxUHRk31pojrnIGVfR1qCpdY15YwNVtRa7RekvdnIWSiMQH5i39VPHWSGlQtMEAIEi0Uomyjl9v+Kc= ceph-9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:56:08.205 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-12T22:56:08.513 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:08 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1013862136' entity='client.admin' 2024-04-12T22:56:10.888 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:10 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:10.888 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:10 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd/host:smithi144", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:56:10.888 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:10 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:10.888 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:10 smithi144 ceph-mon[24815]: Deploying daemon alertmanager.smithi144 on smithi144 2024-04-12T22:56:12.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:12 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2752206732' entity='client.admin' 2024-04-12T22:56:12.408 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-12T22:56:12.408 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-12T22:56:15.920 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi181 2024-04-12T22:56:15.920 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-04-12T22:56:15.920 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.conf 2024-04-12T22:56:15.938 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-04-12T22:56:15.938 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:56:15.996 INFO:tasks.cephadm:Adding host smithi181 to orchestrator... 2024-04-12T22:56:15.997 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch host add smithi181 2024-04-12T22:56:16.022 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:15 smithi144 ceph-mon[24815]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:56:16.022 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:15 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:20.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:19 smithi144 ceph-mon[24815]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:20.476 INFO:teuthology.orchestra.run.smithi144.stdout:Added host 'smithi181' with addr '172.21.15.181' 2024-04-12T22:56:20.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:20 smithi144 ceph-mon[24815]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:56:20.832 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:20 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:21.097 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch host ls --format=json 2024-04-12T22:56:21.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:21 smithi144 ceph-mon[24815]: Deploying cephadm binary to smithi181 2024-04-12T22:56:21.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:21 smithi144 ceph-mon[24815]: Added host smithi181 2024-04-12T22:56:21.847 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:21 smithi144 ceph-mon[24815]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:23.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:22 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:23.006 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:22 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:56:23.006 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:22 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T22:56:23.006 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:22 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:56:23.006 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:22 smithi144 ceph-mon[24815]: Deploying daemon crash.smithi144 on smithi144 2024-04-12T22:56:23.006 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:22 smithi144 ceph-mon[24815]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:24.278 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T22:56:24.278 INFO:teuthology.orchestra.run.smithi144.stdout:[{"addr": "172.21.15.144", "hostname": "smithi144", "labels": [], "status": ""}, {"addr": "172.21.15.181", "hostname": "smithi181", "labels": [], "status": ""}] 2024-04-12T22:56:24.889 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:24 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:24.917 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-12T22:56:24.917 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd crush tunables default 2024-04-12T22:56:26.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T22:56:26.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:26.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:26.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:26.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:26.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T22:56:26.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:25 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:26.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:26 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T22:56:26.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:26 smithi144 ceph-mon[24815]: Deploying daemon grafana.smithi144 on smithi144 2024-04-12T22:56:27.559 INFO:teuthology.orchestra.run.smithi144.stderr:adjusted tunables profile to default 2024-04-12T22:56:27.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:27 smithi144 ceph-mon[24815]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:27.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:27 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3440923834' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T22:56:28.188 INFO:tasks.cephadm:Adding mon.smithi144 on smithi144 2024-04-12T22:56:28.188 INFO:tasks.cephadm:Adding mon.smithi181 on smithi181 2024-04-12T22:56:28.189 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch apply mon '2;smithi144:172.21.15.144=smithi144;smithi181:172.21.15.181=smithi181' 2024-04-12T22:56:29.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:28 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3440923834' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T22:56:29.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:28 smithi144 ceph-mon[24815]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T22:56:29.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:28 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:30.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:29 smithi144 ceph-mon[24815]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:30.060 INFO:teuthology.orchestra.run.smithi181.stdout:Scheduled mon update... 2024-04-12T22:56:30.642 DEBUG:teuthology.orchestra.run.smithi181:mon.smithi181> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi181.service 2024-04-12T22:56:30.645 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:30.645 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:31 smithi144 ceph-mon[24815]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi144:172.21.15.144=smithi144;smithi181:172.21.15.181=smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:56:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:31 smithi144 ceph-mon[24815]: Saving service mon spec with placement smithi144:172.21.15.144=smithi144;smithi181:172.21.15.181=smithi181;count:2 2024-04-12T22:56:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:31 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:31 smithi144 ceph-mon[24815]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:32.500 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:32.501 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:32.502 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:33.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:32 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/4171823967' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:34.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:33 smithi144 ceph-mon[24815]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:34.096 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:34.097 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:34 smithi144 ceph-mon[24815]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:35.909 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:35.909 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:35.911 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:35 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/4101008706' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:36 smithi144 ceph-mon[24815]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:37.541 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:37.541 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:39.380 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:39.380 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:39.383 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:39.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:39 smithi144 ceph-mon[24815]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:39.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:39 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:39.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:39 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/2645988433' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:40.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:40 smithi144 ceph-mon[24815]: Deploying daemon node-exporter.smithi144 on smithi144 2024-04-12T22:56:40.955 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:40.956 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:41.919 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:41 smithi144 ceph-mon[24815]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:42.793 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:42.793 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:42.795 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:43.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:43 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:43.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:43 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T22:56:43.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:43 smithi144 ceph-mon[24815]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:56:43.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:43 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/122889181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:44.305 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:44 smithi144 ceph-mon[24815]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T22:56:44.305 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:44 smithi144 ceph-mon[24815]: mgrmap e14: smithi144.rlueky(active, since 44s) 2024-04-12T22:56:44.371 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:44.372 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:46.111 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:46.112 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:46.114 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:46.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:46 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/2220221144' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:47.734 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:47.734 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:49.445 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:49.445 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:49.448 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: Active manager daemon smithi144.rlueky restarted 2024-04-12T22:56:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: Activating manager daemon smithi144.rlueky 2024-04-12T22:56:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T22:56:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/3085579281' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: mgrmap e15: smithi144.rlueky(active, starting, since 0.694502s) 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: Manager daemon smithi144.rlueky is now available 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:56:50.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:56:51.003 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:51.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: [12/Apr/2024:22:56:50] ENGINE Bus STARTING 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: [12/Apr/2024:22:56:50] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: [12/Apr/2024:22:56:50] ENGINE Bus STARTED 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:51.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:51 smithi144 ceph-mon[24815]: mgrmap e16: smithi144.rlueky(active, since 1.6964s) 2024-04-12T22:56:52.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:52 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:52.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:52 smithi144 ceph-mon[24815]: mgrmap e17: smithi144.rlueky(active, since 2s) 2024-04-12T22:56:52.770 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:52.770 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:52.772 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:53.641 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:53 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/3557552042' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:53.641 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:53 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd/host:smithi144", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:56:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:54 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T22:56:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:54 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:56:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:55.065 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:55.065 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:56:57.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:57.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:57.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:56:57.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:56:57.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:56:57.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T22:56:57.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:56:57.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:56 smithi144 ceph-mon[24815]: Deploying daemon crash.smithi181 on smithi181 2024-04-12T22:56:57.327 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:56:57.327 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:56:57.329 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:56:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:56:57 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/3679133114' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:56:59.186 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:56:59.186 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:57:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T22:57:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T22:57:01.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T22:57:01.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:01.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:00 smithi144 ceph-mon[24815]: Deploying daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T22:57:01.509 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:57:01.509 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:57:01.511 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:57:02.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:01 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/3119539687' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:04.068 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:57:04.069 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:57:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:04 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:04 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T22:57:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:04 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:04 smithi144 ceph-mon[24815]: Deploying daemon mon.smithi181 on smithi181 2024-04-12T22:57:06.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:07.755 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:07 smithi181 podman[26888]: 2024-04-12 22:57:07.646363884 +0000 UTC m=+0.018836828 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-12T22:57:07.755 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:07 smithi181 podman[26888]: 2024-04-12 22:57:07.765359842 +0000 UTC m=+0.137832773 container create 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, ceph=True, io.openshift.expose-services=, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64) 2024-04-12T22:57:07.861 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:57:07.861 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":1,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:54:51.314654Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-12T22:57:07.863 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 1 2024-04-12T22:57:08.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:07 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/803759594' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 podman[26888]: 2024-04-12 22:57:08.646603552 +0000 UTC m=+1.019076475 container init 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, distribution-scope=public, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, RELEASE=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True) 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 podman[26888]: 2024-04-12 22:57:08.65010734 +0000 UTC m=+1.022580264 container start 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_CLEAN=True, version=8) 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pidfile_write: ignore empty --pid-file 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: load: jerasure load: lrc 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: RocksDB version: 6.15.5 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-12T22:57:08.762 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Compile date Apr 18 2022 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: DB SUMMARY 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: DB Session ID: OKGI4W04DF62MP0LI7XV 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: CURRENT file: CURRENT 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi181/store.db dir, Total Num: 0, files: 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi181/store.db: 000004.log size: 511 ; 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.error_if_exists: 0 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.create_if_missing: 0 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.paranoid_checks: 1 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.env: 0x5641a80c6860 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.fs: Posix File System 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.info_log: 0x5641a8f45fa0 2024-04-12T22:57:08.763 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.statistics: (nil) 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.use_fsync: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_log_file_size: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.allow_fallocate: 1 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.use_direct_reads: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.db_log_dir: 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi181/store.db 2024-04-12T22:57:08.764 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T22:57:08.765 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T22:57:08.765 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T22:57:08.765 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T22:57:08.765 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T22:57:08.765 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.write_buffer_manager: 0x5641a90362d0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T22:57:08.767 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.unordered_write: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.row_cache: None 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.wal_filter: None 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.preserve_deletes: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.two_write_queues: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.atomic_flush: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T22:57:08.768 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.log_readahead_size: 0 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_background_jobs: 2 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_background_compactions: -1 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_subcompactions: 1 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T22:57:08.769 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_open_files: -1 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_background_flushes: -1 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Compression algorithms supported: 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kZSTD supported: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kXpressCompression supported: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kLZ4Compression supported: 1 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kBZip2Compression supported: 0 2024-04-12T22:57:08.770 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kZlibCompression supported: 1 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: kSnappyCompression supported: 1 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000003 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.merge_operator: 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_filter: None 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5641a8f11d00) 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-04-12T22:57:08.771 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_type: 0 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_index_type: 0 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_shortening: 1 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: hash_index_allow_collision: 1 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: checksum: 1 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: no_block_cache: 0 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache: 0x5641a8f7c170 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_options: 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: capacity : 536870912 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: num_shard_bits : 4 2024-04-12T22:57:08.772 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: strict_capacity_limit : 0 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_compressed: (nil) 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: persistent_cache: (nil) 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size: 4096 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size_deviation: 10 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_restart_interval: 16 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_block_restart_interval: 1 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: metadata_block_size: 4096 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: partition_filters: 0 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: use_delta_encoding: 1 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: whole_key_filtering: 1 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: verify_compression: 0 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: format_version: 4 2024-04-12T22:57:08.773 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_index_compression: 1 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_align: 0 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression: NoCompression 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.num_levels: 7 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T22:57:08.774 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T22:57:08.775 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T22:57:08.776 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.arena_block_size: 4194304 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T22:57:08.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.table_properties_collectors: 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.inplace_update_support: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.bloom_locality: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.max_successive_merges: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.ttl: 2592000 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T22:57:08.778 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.enable_blob_files: false 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.min_blob_size: 0 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712962628675293, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712962628675589, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712962628, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "93990e26-c5cb-48f7-9d10-0762aee15012", "db_session_id": "OKGI4W04DF62MP0LI7XV"}} 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712962628675891, "job": 1, "event": "recovery_finished"} 2024-04-12T22:57:08.779 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi181/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5641a8f62a80 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: DB pointer 0x5641a8f72000 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** DB Stats ** 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.780 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T22:57:08.781 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.782 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T22:57:08.783 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181 does not exist in monmap, will attempt to join an existing cluster 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: using public_addr v2:172.21.15.181:0/0 -> [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: starting mon.smithi181 rank -1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi181 fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(???) e0 preinit fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).mds e1 new map 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).mds e1 print_map 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: e1 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: legacy client fscid: -1 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: No filesystems configured 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-12T22:57:08.784 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying cephadm binary to smithi181 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Added host smithi181 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying daemon crash.smithi144 on smithi144 2024-04-12T22:57:08.785 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying daemon grafana.smithi144 on smithi144 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3440923834' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3440923834' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: osdmap e4: 0 total, 0 up, 0 in 2024-04-12T22:57:08.786 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi144:172.21.15.144=smithi144;smithi181:172.21.15.181=smithi181", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Saving service mon spec with placement smithi144:172.21.15.144=smithi144;smithi181:172.21.15.181=smithi181;count:2 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/4171823967' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/4101008706' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/2645988433' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.787 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying daemon node-exporter.smithi144 on smithi144 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/122889181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14162 172.21.15.144:0/3138854011' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mgrmap e14: smithi144.rlueky(active, since 44s) 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/2220221144' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Active manager daemon smithi144.rlueky restarted 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Activating manager daemon smithi144.rlueky 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/3085579281' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mgrmap e15: smithi144.rlueky(active, starting, since 0.694502s) 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:57:08.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Manager daemon smithi144.rlueky is now available 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:57:08.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: [12/Apr/2024:22:56:50] ENGINE Bus STARTING 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: [12/Apr/2024:22:56:50] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: [12/Apr/2024:22:56:50] ENGINE Bus STARTED 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mgrmap e16: smithi144.rlueky(active, since 1.6964s) 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mgrmap e17: smithi144.rlueky(active, since 2s) 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/3557552042' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd/host:smithi144", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:57:08.790 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying daemon crash.smithi181 on smithi181 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/3679133114' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-12T22:57:08.791 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/3119539687' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: Deploying daemon mon.smithi181 on smithi181 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/803759594' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 ceph-mon[26930]: mon.smithi181@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 bash[26888]: 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c 2024-04-12T22:57:08.792 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:08 smithi181 systemd[1]: Started Ceph mon.smithi181 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T22:57:09.533 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-12T22:57:09.533 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-12T22:57:14.020 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: mon.smithi144 calling monitor election 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: mon.smithi181 calling monitor election 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.021 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: mon.smithi144 is new leader, mons smithi144,smithi181 in quorum (ranks 0,1) 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: monmap e2: 2 mons at {smithi144=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],smithi181=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: fsmap 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: mgrmap e17: smithi144.rlueky(active, since 24s) 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz started 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: overall HEALTH_OK 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:14.022 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:13 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T22:57:14.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:57:14.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: mon.smithi144 calling monitor election 2024-04-12T22:57:14.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: mon.smithi181 calling monitor election 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: mon.smithi144 is new leader, mons smithi144,smithi181 in quorum (ranks 0,1) 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: monmap e2: 2 mons at {smithi144=[v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0],smithi181=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: fsmap 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: osdmap e5: 0 total, 0 up, 0 in 2024-04-12T22:57:14.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: mgrmap e17: smithi144.rlueky(active, since 24s) 2024-04-12T22:57:14.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: Standby manager daemon smithi181.whsrmz started 2024-04-12T22:57:14.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T22:57:14.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: overall HEALTH_OK 2024-04-12T22:57:14.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T22:57:14.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:14.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:13 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/763291599' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T22:57:14.525 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-04-12T22:57:14.525 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":2,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","modified":"2024-04-12T22:57:08.700854Z","created":"2024-04-12T22:54:51.314654Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi144","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:3300","nonce":0},{"type":"v1","addr":"172.21.15.144:6789","nonce":0}]},"addr":"172.21.15.144:6789/0","public_addr":"172.21.15.144:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi181","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-12T22:57:14.526 INFO:teuthology.orchestra.run.smithi181.stderr:dumped monmap epoch 2 2024-04-12T22:57:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: Deploying daemon node-exporter.smithi181 on smithi181 2024-04-12T22:57:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: mgrmap e18: smithi144.rlueky(active, since 24s), standbys: smithi181.whsrmz 2024-04-12T22:57:15.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T22:57:15.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/123017532' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:15.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:15.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:15.107 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:15.107 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: Deploying daemon node-exporter.smithi181 on smithi181 2024-04-12T22:57:15.107 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: mgrmap e18: smithi144.rlueky(active, since 24s), standbys: smithi181.whsrmz 2024-04-12T22:57:15.107 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T22:57:15.107 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/123017532' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-12T22:57:15.107 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:57:15.108 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:15.130 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-12T22:57:15.130 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph config generate-minimal-conf 2024-04-12T22:57:16.835 INFO:teuthology.orchestra.run.smithi144.stdout:# minimal ceph.conf for 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:57:16.835 INFO:teuthology.orchestra.run.smithi144.stdout:[global] 2024-04-12T22:57:16.835 INFO:teuthology.orchestra.run.smithi144.stdout: fsid = 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T22:57:16.835 INFO:teuthology.orchestra.run.smithi144.stdout: mon_host = [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] 2024-04-12T22:57:16.835 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:16 smithi144 ceph-mon[24815]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:17.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:16 smithi181 ceph-mon[26930]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:17.436 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-12T22:57:17.436 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:57:17.436 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T22:57:17.464 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:57:17.465 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:57:17.537 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-04-12T22:57:17.537 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.conf 2024-04-12T22:57:17.567 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-04-12T22:57:17.568 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:57:17.635 INFO:tasks.cephadm:Deploying OSDs... 2024-04-12T22:57:17.635 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T22:57:17.635 DEBUG:teuthology.orchestra.run.smithi144:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T22:57:17.651 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T22:57:17.651 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_1 2024-04-12T22:57:17.706 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T22:57:17.706 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:17.706 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-12T22:57:17.707 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:17.707 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:17.707 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-04-12 22:56:40.022477877 +0000 2024-04-12T22:57:17.707 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-04-12 22:53:56.764898056 +0000 2024-04-12T22:57:17.707 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-04-12 22:53:56.764898056 +0000 2024-04-12T22:57:17.707 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-04-12 22:53:56.764898056 +0000 2024-04-12T22:57:17.707 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T22:57:17.771 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-04-12T22:57:17.771 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-04-12T22:57:17.771 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000213762 s, 2.4 MB/s 2024-04-12T22:57:17.772 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T22:57:17.829 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_2 2024-04-12T22:57:17.883 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-04-12 22:56:40.023477860 +0000 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-04-12 22:53:57.102891984 +0000 2024-04-12T22:57:17.884 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-04-12 22:53:57.102891984 +0000 2024-04-12T22:57:17.885 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-04-12 22:53:57.102891984 +0000 2024-04-12T22:57:17.885 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T22:57:17.948 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:17 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2860361297' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:17.948 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:17 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:17.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:17 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2860361297' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:17.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:17 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:17.950 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-04-12T22:57:17.950 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-04-12T22:57:17.950 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.00026487 s, 1.9 MB/s 2024-04-12T22:57:17.951 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T22:57:18.008 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_3 2024-04-12T22:57:18.063 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T22:57:18.063 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:18.063 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-12T22:57:18.063 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:18.063 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:18.064 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-04-12 22:56:40.023477860 +0000 2024-04-12T22:57:18.064 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-04-12 22:53:57.431886074 +0000 2024-04-12T22:57:18.064 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-04-12 22:53:57.431886074 +0000 2024-04-12T22:57:18.064 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-04-12 22:53:57.431886074 +0000 2024-04-12T22:57:18.064 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T22:57:18.129 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-04-12T22:57:18.129 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-04-12T22:57:18.129 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000221814 s, 2.3 MB/s 2024-04-12T22:57:18.130 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T22:57:18.185 DEBUG:teuthology.orchestra.run.smithi144:> stat /dev/vg_nvme/lv_4 2024-04-12T22:57:18.240 INFO:teuthology.orchestra.run.smithi144.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T22:57:18.240 INFO:teuthology.orchestra.run.smithi144.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout:Access: 2024-04-12 22:56:40.023477860 +0000 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout:Modify: 2024-04-12 22:53:57.805879356 +0000 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout:Change: 2024-04-12 22:53:57.805879356 +0000 2024-04-12T22:57:18.241 INFO:teuthology.orchestra.run.smithi144.stdout: Birth: 2024-04-12 22:53:57.805879356 +0000 2024-04-12T22:57:18.241 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T22:57:18.306 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records in 2024-04-12T22:57:18.306 INFO:teuthology.orchestra.run.smithi144.stderr:1+0 records out 2024-04-12T22:57:18.306 INFO:teuthology.orchestra.run.smithi144.stderr:512 bytes copied, 0.000194089 s, 2.6 MB/s 2024-04-12T22:57:18.307 DEBUG:teuthology.orchestra.run.smithi144:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T22:57:18.362 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-04-12T22:57:18.363 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/scratch_devs of=/dev/stdout 2024-04-12T22:57:18.383 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-12T22:57:18.383 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_1 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-04-12 22:57:14.319093989 +0000 2024-04-12T22:57:18.440 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-04-12 22:53:56.758880614 +0000 2024-04-12T22:57:18.441 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-04-12 22:53:56.758880614 +0000 2024-04-12T22:57:18.441 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-04-12 22:53:56.758880614 +0000 2024-04-12T22:57:18.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-12T22:57:18.509 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-04-12T22:57:18.509 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-04-12T22:57:18.509 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000202035 s, 2.5 MB/s 2024-04-12T22:57:18.510 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-12T22:57:18.568 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_2 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-04-12 22:57:14.319093989 +0000 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-04-12 22:53:57.121873873 +0000 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-04-12 22:53:57.121873873 +0000 2024-04-12T22:57:18.625 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-04-12 22:53:57.121873873 +0000 2024-04-12T22:57:18.625 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-12T22:57:18.692 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-04-12T22:57:18.692 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-04-12T22:57:18.692 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000195732 s, 2.6 MB/s 2024-04-12T22:57:18.693 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-12T22:57:18.750 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_3 2024-04-12T22:57:18.806 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-12T22:57:18.806 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:18.806 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-12T22:57:18.806 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:18.806 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:18.806 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-04-12 22:57:14.319093989 +0000 2024-04-12T22:57:18.807 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-04-12 22:53:57.462867541 +0000 2024-04-12T22:57:18.807 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-04-12 22:53:57.462867541 +0000 2024-04-12T22:57:18.807 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-04-12 22:53:57.462867541 +0000 2024-04-12T22:57:18.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-12T22:57:18.872 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-04-12T22:57:18.873 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-04-12T22:57:18.873 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000197489 s, 2.6 MB/s 2024-04-12T22:57:18.875 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-12T22:57:18.931 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_4 2024-04-12T22:57:18.985 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-04-12 22:57:14.319093989 +0000 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-04-12 22:53:57.820860893 +0000 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-04-12 22:53:57.820860893 +0000 2024-04-12T22:57:18.986 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-04-12 22:53:57.820860893 +0000 2024-04-12T22:57:18.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-12T22:57:19.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:18 smithi144 ceph-mon[24815]: Deploying daemon prometheus.smithi144 on smithi144 2024-04-12T22:57:19.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:18 smithi144 ceph-mon[24815]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:19.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:18 smithi181 ceph-mon[26930]: Deploying daemon prometheus.smithi144 on smithi144 2024-04-12T22:57:19.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:18 smithi181 ceph-mon[26930]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:19.051 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-04-12T22:57:19.051 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-04-12T22:57:19.051 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000203643 s, 2.5 MB/s 2024-04-12T22:57:19.052 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-12T22:57:19.108 INFO:tasks.cephadm:Deploying osd.0 on smithi144 with /dev/vg_nvme/lv_4... 2024-04-12T22:57:19.108 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T22:57:21.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:20 smithi144 ceph-mon[24815]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:21.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:21.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:20 smithi181 ceph-mon[26930]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:21.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:22.372 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T22:57:22.372 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T22:57:22.372 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2024-04-12T22:57:22.372 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2024-04-12T22:57:22.372 INFO:teuthology.orchestra.run.smithi144.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0132565 s, 791 MB/s 2024-04-12T22:57:22.372 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2024-04-12T22:57:23.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:22 smithi144 ceph-mon[24815]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:23.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:22 smithi181 ceph-mon[26930]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:23.965 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_4 2024-04-12T22:57:24.802 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:24 smithi144 ceph-mon[24815]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:25.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:24 smithi181 ceph-mon[26930]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:26.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:26 smithi144 ceph-mon[24815]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:27.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:26 smithi181 ceph-mon[26930]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:27 smithi144 ceph-mon[24815]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:57:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:57:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:57:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:27 smithi181 ceph-mon[26930]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:57:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:57:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:57:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:29.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:28 smithi144 ceph-mon[24815]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:29.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:28 smithi181 ceph-mon[26930]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/668310506' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dd4c61f1-5d94-4c3b-87e1-2486e2858ba7"}]: dispatch 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/668310506' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dd4c61f1-5d94-4c3b-87e1-2486e2858ba7"}]': finished 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T22:57:31.122 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:31.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:31.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:31.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:57:31.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:31.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:57:31.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/668310506' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dd4c61f1-5d94-4c3b-87e1-2486e2858ba7"}]: dispatch 2024-04-12T22:57:31.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/668310506' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dd4c61f1-5d94-4c3b-87e1-2486e2858ba7"}]': finished 2024-04-12T22:57:31.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: osdmap e6: 1 total, 0 up, 1 in 2024-04-12T22:57:31.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:32.117 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:31 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3840089531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:57:32.117 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:31 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:32.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:31 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3840089531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:57:32.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:31 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:33.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:33 smithi144 ceph-mon[24815]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T22:57:33.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:33 smithi144 ceph-mon[24815]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:33.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:33 smithi144 ceph-mon[24815]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:57:33.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:33.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:33 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T22:57:33.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:33 smithi181 ceph-mon[26930]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:33.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:33 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T22:57:33.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:34.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:34.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T22:57:34.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:34.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:34.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:34.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: Reconfiguring mon.smithi144 (unknown last config time)... 2024-04-12T22:57:34.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T22:57:34.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T22:57:34.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:34.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:34 smithi181 ceph-mon[26930]: Reconfiguring daemon mon.smithi144 on smithi144 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: Reconfiguring mon.smithi144 (unknown last config time)... 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T22:57:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:35.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:34 smithi144 ceph-mon[24815]: Reconfiguring daemon mon.smithi144 on smithi144 2024-04-12T22:57:36.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:36.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:36.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T22:57:36.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T22:57:36.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:36.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:36.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:36.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T22:57:36.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T22:57:36.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:37.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:36 smithi144 ceph-mon[24815]: Reconfiguring mgr.smithi144.rlueky (unknown last config time)... 2024-04-12T22:57:37.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:36 smithi144 ceph-mon[24815]: Reconfiguring daemon mgr.smithi144.rlueky on smithi144 2024-04-12T22:57:37.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:36 smithi144 ceph-mon[24815]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:37.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:36 smithi181 ceph-mon[26930]: Reconfiguring mgr.smithi144.rlueky (unknown last config time)... 2024-04-12T22:57:37.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:36 smithi181 ceph-mon[26930]: Reconfiguring daemon mgr.smithi144.rlueky on smithi144 2024-04-12T22:57:37.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:36 smithi181 ceph-mon[26930]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:38.736 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:38.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:38 smithi144 ceph-mon[24815]: Reconfiguring alertmanager.smithi144 (dependencies changed)... 2024-04-12T22:57:38.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:38 smithi144 ceph-mon[24815]: Reconfiguring daemon alertmanager.smithi144 on smithi144 2024-04-12T22:57:38.737 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:38 smithi144 ceph-mon[24815]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:38.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:38.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:38 smithi181 ceph-mon[26930]: Reconfiguring alertmanager.smithi144 (dependencies changed)... 2024-04-12T22:57:38.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:38 smithi181 ceph-mon[26930]: Reconfiguring daemon alertmanager.smithi144 on smithi144 2024-04-12T22:57:38.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:38 smithi181 ceph-mon[26930]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:39.965 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:39 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T22:57:39.965 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:39 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:39.965 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:39 smithi181 ceph-mon[26930]: Deploying daemon osd.0 on smithi144 2024-04-12T22:57:40.002 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:39 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T22:57:40.002 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:39 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:40.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:39 smithi144 ceph-mon[24815]: Deploying daemon osd.0 on smithi144 2024-04-12T22:57:41.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:40 smithi144 ceph-mon[24815]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:41.090 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:40 smithi181 ceph-mon[26930]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:42.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:42 smithi144 ceph-mon[24815]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:43.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:42 smithi181 ceph-mon[26930]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:44.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:44.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:44 smithi144 ceph-mon[24815]: Reconfiguring crash.smithi144 (monmap changed)... 2024-04-12T22:57:44.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:57:44.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:44.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:44 smithi144 ceph-mon[24815]: Reconfiguring daemon crash.smithi144 on smithi144 2024-04-12T22:57:44.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:44 smithi144 ceph-mon[24815]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:44 smithi181 ceph-mon[26930]: Reconfiguring crash.smithi144 (monmap changed)... 2024-04-12T22:57:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:57:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:44 smithi181 ceph-mon[26930]: Reconfiguring daemon crash.smithi144 on smithi144 2024-04-12T22:57:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:44 smithi181 ceph-mon[26930]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:47.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:46 smithi144 ceph-mon[24815]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:47.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:46 smithi181 ceph-mon[26930]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:49.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:48 smithi144 ceph-mon[24815]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:49.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:48 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:49.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:48 smithi181 ceph-mon[26930]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:49.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:48 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:50.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:50.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:50 smithi144 ceph-mon[24815]: Reconfiguring grafana.smithi144 (dependencies changed)... 2024-04-12T22:57:50.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:50 smithi144 ceph-mon[24815]: Reconfiguring daemon grafana.smithi144 on smithi144 2024-04-12T22:57:50.459 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:50 smithi144 ceph-mon[24815]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:50.460 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:57:50.460 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:57:50.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:50.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:50 smithi181 ceph-mon[26930]: Reconfiguring grafana.smithi144 (dependencies changed)... 2024-04-12T22:57:50.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:50 smithi181 ceph-mon[26930]: Reconfiguring daemon grafana.smithi144 on smithi144 2024-04-12T22:57:50.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:50 smithi181 ceph-mon[26930]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:50.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:57:50.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:57:51.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:51 smithi181 ceph-mon[26930]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T22:57:51.729 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:51 smithi144 ceph-mon[24815]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:57:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: osdmap e8: 1 total, 0 up, 1 in 2024-04-12T22:57:52.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:52 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:52.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: osdmap e7: 1 total, 0 up, 1 in 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: from='osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: osdmap e8: 1 total, 0 up, 1 in 2024-04-12T22:57:52.715 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:52 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:53.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:53 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:53.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:53 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:53.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:53 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:53.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:53 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:54.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534] boot 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: Reconfiguring crash.smithi181 (monmap changed)... 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:54.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:54 smithi144 ceph-mon[24815]: Reconfiguring daemon crash.smithi181 on smithi181 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: osd.0 [v2:172.21.15.144:6802/4186511534,v1:172.21.15.144:6803/4186511534] boot 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: osdmap e9: 1 total, 1 up, 1 in 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-12T22:57:54.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:54.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: Reconfiguring crash.smithi181 (monmap changed)... 2024-04-12T22:57:54.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T22:57:54.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:54.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:54 smithi181 ceph-mon[26930]: Reconfiguring daemon crash.smithi181 on smithi181 2024-04-12T22:57:55.444 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 0 on host 'smithi144' 2024-04-12T22:57:56.100 DEBUG:teuthology.orchestra.run.smithi144:osd.0> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.0.service 2024-04-12T22:57:56.103 INFO:tasks.cephadm:Deploying osd.1 on smithi144 with /dev/vg_nvme/lv_3... 2024-04-12T22:57:56.103 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T22:57:56.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:56 smithi181 ceph-mon[26930]: osdmap e10: 1 total, 1 up, 1 in 2024-04-12T22:57:56.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:56.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:56.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T22:57:56.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T22:57:56.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:56.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:56 smithi144 ceph-mon[24815]: osdmap e10: 1 total, 1 up, 1 in 2024-04-12T22:57:56.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:56.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:56.436 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T22:57:56.436 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T22:57:56.436 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: Reconfiguring mgr.smithi181.whsrmz (monmap changed)... 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: Reconfiguring daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T22:57:57.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: Reconfiguring mgr.smithi181.whsrmz (monmap changed)... 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: Reconfiguring daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T22:57:57.357 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:58.224 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:58 smithi144 ceph-mon[24815]: Reconfiguring mon.smithi181 (monmap changed)... 2024-04-12T22:57:58.224 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:58 smithi144 ceph-mon[24815]: Reconfiguring daemon mon.smithi181 on smithi181 2024-04-12T22:57:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:58 smithi181 ceph-mon[26930]: Reconfiguring mon.smithi181 (monmap changed)... 2024-04-12T22:57:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:58 smithi181 ceph-mon[26930]: Reconfiguring daemon mon.smithi181 on smithi181 2024-04-12T22:57:59.395 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:57:59.395 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.395 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T22:57:59.395 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.144:9093"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.144:9093"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.144:3000"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.144:3000"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.144:9095"}]: dispatch 2024-04-12T22:57:59.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.144:9095"}]: dispatch 2024-04-12T22:57:59.397 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.397 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:57:59.397 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:59.397 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:57:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:57:59.475 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.144:9093"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.144:9093"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.144:3000"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.144:3000"}]: dispatch 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.476 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.144:9095"}]: dispatch 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.144:9095"}]: dispatch 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:57:59.477 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:57:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:57:59.500 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T22:57:59.500 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T22:57:59.500 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2024-04-12T22:57:59.500 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2024-04-12T22:57:59.501 INFO:teuthology.orchestra.run.smithi144.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113665 s, 923 MB/s 2024-04-12T22:57:59.501 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2024-04-12T22:58:00.185 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_3 2024-04-12T22:58:00.924 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:00 smithi144 ceph-mon[24815]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:00.925 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:01.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:00 smithi181 ceph-mon[26930]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:01.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:00 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:03.171 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:02 smithi144 ceph-mon[24815]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:03.171 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:58:03.171 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:58:03.171 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:03.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:02 smithi181 ceph-mon[26930]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:03.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:58:03.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:58:03.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:03.979 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:03 smithi144 ceph-mon[24815]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:58:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:03 smithi181 ceph-mon[26930]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:58:05.190 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:04 smithi144 ceph-mon[24815]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:04 smithi181 ceph-mon[26930]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:06.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:05 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/813758315' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "115fcd46-373b-42b7-9598-a226a86ac8ec"}]: dispatch 2024-04-12T22:58:06.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:05 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/813758315' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "115fcd46-373b-42b7-9598-a226a86ac8ec"}]': finished 2024-04-12T22:58:06.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:05 smithi144 ceph-mon[24815]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T22:58:06.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:05 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/813758315' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "115fcd46-373b-42b7-9598-a226a86ac8ec"}]: dispatch 2024-04-12T22:58:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:05 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/813758315' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "115fcd46-373b-42b7-9598-a226a86ac8ec"}]': finished 2024-04-12T22:58:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:05 smithi181 ceph-mon[26930]: osdmap e11: 2 total, 1 up, 2 in 2024-04-12T22:58:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:07.118 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:06 smithi144 ceph-mon[24815]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:07.119 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:06 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3966631331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:58:07.119 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:06 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:07.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:06 smithi181 ceph-mon[26930]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:07.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:06 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3966631331' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:58:07.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:06 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:09.136 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:08 smithi181 ceph-mon[26930]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:09.158 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:08 smithi144 ceph-mon[24815]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:10.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:10 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:11.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:11.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:11.192 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:11.193 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:11.193 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:11.193 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:11.193 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:11.193 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:10 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:12.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:11 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi144 2024-04-12T22:58:12.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:11 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi144 to 12864M 2024-04-12T22:58:12.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:11 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi144 2024-04-12T22:58:12.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:11 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi144 to 12864M 2024-04-12T22:58:13.189 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:12 smithi144 ceph-mon[24815]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:13.234 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:12 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:13.234 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:12 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T22:58:13.234 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:12 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:13.235 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:12 smithi181 ceph-mon[26930]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:13.235 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:12 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:13.235 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:12 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T22:58:13.235 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:12 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:14.052 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:13 smithi144 ceph-mon[24815]: Deploying daemon osd.1 on smithi144 2024-04-12T22:58:14.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:13 smithi181 ceph-mon[26930]: Deploying daemon osd.1 on smithi144 2024-04-12T22:58:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:14 smithi144 ceph-mon[24815]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:15.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:14 smithi181 ceph-mon[26930]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:16 smithi181 ceph-mon[26930]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:17.203 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:16 smithi144 ceph-mon[24815]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:18.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:18 smithi144 ceph-mon[24815]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:18.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:18.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:19.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:18 smithi181 ceph-mon[26930]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:19.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:19.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:20.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:20.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:20.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:20.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:20.541 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:20.541 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:20.541 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:20.541 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:21.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:21 smithi144 ceph-mon[24815]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:21.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:21 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:21.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:21 smithi181 ceph-mon[26930]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:21.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:21 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:21.936 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 1 on host 'smithi144' 2024-04-12T22:58:22.590 DEBUG:teuthology.orchestra.run.smithi144:osd.1> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.1.service 2024-04-12T22:58:22.592 INFO:tasks.cephadm:Deploying osd.2 on smithi144 with /dev/vg_nvme/lv_2... 2024-04-12T22:58:22.593 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T22:58:23.148 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:22 smithi144 ceph-mon[24815]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:23.148 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:22 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:23.149 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:22 smithi144 ceph-mon[24815]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T22:58:23.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:22 smithi181 ceph-mon[26930]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:23.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:22 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:23.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:22 smithi181 ceph-mon[26930]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T22:58:24.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-mon[24815]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T22:58:24.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-mon[24815]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T22:58:24.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:24.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-mon[24815]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:58:24.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-mon[24815]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:24.276 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[39332]: 2024-04-12T22:58:24.225+0000 7f48efc2c700 -1 osd.1 0 waiting for initial osdmap 2024-04-12T22:58:24.276 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:24 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[39332]: 2024-04-12T22:58:24.228+0000 7f48eadc4700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T22:58:24.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:24 smithi181 ceph-mon[26930]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T22:58:24.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:24 smithi181 ceph-mon[26930]: osdmap e12: 2 total, 1 up, 2 in 2024-04-12T22:58:24.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:24.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:24 smithi181 ceph-mon[26930]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:58:24.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:24 smithi181 ceph-mon[26930]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-12T22:58:25.003 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T22:58:25.003 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T22:58:25.003 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2024-04-12T22:58:25.003 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2024-04-12T22:58:25.003 INFO:teuthology.orchestra.run.smithi144.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.014134 s, 742 MB/s 2024-04-12T22:58:25.004 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2024-04-12T22:58:25.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:25 smithi144 ceph-mon[24815]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:58:25.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:25 smithi144 ceph-mon[24815]: osdmap e13: 2 total, 1 up, 2 in 2024-04-12T22:58:25.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:25 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:25.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:25 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:25.671 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_2 2024-04-12T22:58:25.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:25 smithi181 ceph-mon[26930]: from='osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:58:25.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:25 smithi181 ceph-mon[26930]: osdmap e13: 2 total, 1 up, 2 in 2024-04-12T22:58:25.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:25 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:25.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:25 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:26.374 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:26 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T22:58:26.375 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:26 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T22:58:26.375 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:26 smithi144 ceph-mon[24815]: osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096] boot 2024-04-12T22:58:26.375 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:26 smithi144 ceph-mon[24815]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T22:58:26.375 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:26 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:26.375 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:26 smithi144 ceph-mon[24815]: pgmap v50: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:26.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:26 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T22:58:26.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:26 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T22:58:26.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:26 smithi181 ceph-mon[26930]: osd.1 [v2:172.21.15.144:6810/73710096,v1:172.21.15.144:6811/73710096] boot 2024-04-12T22:58:26.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:26 smithi181 ceph-mon[26930]: osdmap e14: 2 total, 2 up, 2 in 2024-04-12T22:58:26.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:26 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T22:58:26.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:26 smithi181 ceph-mon[26930]: pgmap v50: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:27.433 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:27 smithi144 ceph-mon[24815]: osdmap e15: 2 total, 2 up, 2 in 2024-04-12T22:58:27.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:27 smithi181 ceph-mon[26930]: osdmap e15: 2 total, 2 up, 2 in 2024-04-12T22:58:28.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:28 smithi144 ceph-mon[24815]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:58:28.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:28 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:58:28.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:28 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:58:28.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:28 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:28.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:28 smithi144 ceph-mon[24815]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:28.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:28 smithi181 ceph-mon[26930]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:58:28.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:28 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:58:28.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:28 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:58:28.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:28 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:28.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:28 smithi181 ceph-mon[26930]: pgmap v52: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:30.039 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:29 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:30.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:29 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:31.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:30 smithi181 ceph-mon[26930]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:31.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3457236293' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "293a464f-f039-4f0f-8158-1111cf1760df"}]: dispatch 2024-04-12T22:58:31.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3457236293' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "293a464f-f039-4f0f-8158-1111cf1760df"}]': finished 2024-04-12T22:58:31.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:30 smithi181 ceph-mon[26930]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T22:58:31.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:31.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:30 smithi144 ceph-mon[24815]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:31.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3457236293' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "293a464f-f039-4f0f-8158-1111cf1760df"}]: dispatch 2024-04-12T22:58:31.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3457236293' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "293a464f-f039-4f0f-8158-1111cf1760df"}]': finished 2024-04-12T22:58:31.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:30 smithi144 ceph-mon[24815]: osdmap e16: 3 total, 2 up, 3 in 2024-04-12T22:58:31.207 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:32.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:31 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3555575412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:58:32.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:31 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3555575412' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:58:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:32 smithi181 ceph-mon[26930]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:32 smithi144 ceph-mon[24815]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:34 smithi144 ceph-mon[24815]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:35.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:34 smithi181 ceph-mon[26930]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:37.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:37.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi144 2024-04-12T22:58:37.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:37.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:37.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:37.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi144 to 6432M 2024-04-12T22:58:37.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:37.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:37.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:37.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:37.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi144 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi144 to 6432M 2024-04-12T22:58:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:37.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:37.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:37.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:37.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:38.185 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:37 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T22:58:38.185 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:37 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:38.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:37 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T22:58:38.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:37 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:39.189 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:38 smithi144 ceph-mon[24815]: Deploying daemon osd.2 on smithi144 2024-04-12T22:58:39.189 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:38 smithi144 ceph-mon[24815]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:39.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:38 smithi181 ceph-mon[26930]: Deploying daemon osd.2 on smithi144 2024-04-12T22:58:39.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:38 smithi181 ceph-mon[26930]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:41.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:40 smithi181 ceph-mon[26930]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:41.210 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:40 smithi144 ceph-mon[24815]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:43.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:42 smithi181 ceph-mon[26930]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:43.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:42 smithi144 ceph-mon[24815]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:44.934 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:45.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:47.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:46 smithi144 ceph-mon[24815]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:47.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:46 smithi181 ceph-mon[26930]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:47.481 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 2 on host 'smithi144' 2024-04-12T22:58:47.969 DEBUG:teuthology.orchestra.run.smithi144:osd.2> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2.service 2024-04-12T22:58:47.973 INFO:tasks.cephadm:Deploying osd.3 on smithi144 with /dev/vg_nvme/lv_1... 2024-04-12T22:58:47.973 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T22:58:48.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:47 smithi144 ceph-mon[24815]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T22:58:48.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:48.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:47 smithi181 ceph-mon[26930]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T22:58:48.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:49.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-mon[24815]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:49.146 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-mon[24815]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T22:58:49.146 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-mon[24815]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T22:58:49.146 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:49.146 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-mon[24815]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:58:49.146 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[43017]: 2024-04-12T22:58:48.791+0000 7ff5e2191700 -1 osd.2 0 waiting for initial osdmap 2024-04-12T22:58:49.146 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:48 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[43017]: 2024-04-12T22:58:48.794+0000 7ff5deb2c700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T22:58:49.147 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:48 smithi181 ceph-mon[26930]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:49.147 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:48 smithi181 ceph-mon[26930]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T22:58:49.147 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:48 smithi181 ceph-mon[26930]: osdmap e17: 3 total, 2 up, 3 in 2024-04-12T22:58:49.147 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:48 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:49.147 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:48 smithi181 ceph-mon[26930]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:58:50.099 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:49 smithi144 ceph-mon[24815]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:58:50.099 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:49 smithi144 ceph-mon[24815]: osdmap e18: 3 total, 2 up, 3 in 2024-04-12T22:58:50.099 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:50.100 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:50.100 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:58:50.100 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:58:50.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:49 smithi181 ceph-mon[26930]: from='osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:58:50.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:49 smithi181 ceph-mon[26930]: osdmap e18: 3 total, 2 up, 3 in 2024-04-12T22:58:50.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:50.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:50.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:58:50.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:58:50.571 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T22:58:50.571 INFO:teuthology.orchestra.run.smithi144.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T22:58:50.571 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10+0 records in 2024-04-12T22:58:50.571 INFO:teuthology.orchestra.run.smithi144.stderr:10+0 records out 2024-04-12T22:58:50.571 INFO:teuthology.orchestra.run.smithi144.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124882 s, 840 MB/s 2024-04-12T22:58:50.571 INFO:teuthology.orchestra.run.smithi144.stderr:--> Zapping successful for: 2024-04-12T22:58:51.139 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:50 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T22:58:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:50 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T22:58:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:50 smithi144 ceph-mon[24815]: pgmap v66: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:50 smithi144 ceph-mon[24815]: osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561] boot 2024-04-12T22:58:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:50 smithi144 ceph-mon[24815]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T22:58:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:51.162 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi144:vg_nvme/lv_1 2024-04-12T22:58:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:50 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T22:58:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:50 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T22:58:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:50 smithi181 ceph-mon[26930]: pgmap v66: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-04-12T22:58:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:50 smithi181 ceph-mon[26930]: osd.2 [v2:172.21.15.144:6818/961664561,v1:172.21.15.144:6819/961664561] boot 2024-04-12T22:58:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:50 smithi181 ceph-mon[26930]: osdmap e19: 3 total, 3 up, 3 in 2024-04-12T22:58:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T22:58:52.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:52 smithi144 ceph-mon[24815]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T22:58:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:52 smithi181 ceph-mon[26930]: osdmap e20: 3 total, 3 up, 3 in 2024-04-12T22:58:53.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:53 smithi144 ceph-mon[24815]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:53.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:53 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T22:58:53.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:53 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T22:58:53.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:53 smithi144 ceph-mon[24815]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T22:58:53.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:53 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T22:58:53.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:53 smithi181 ceph-mon[26930]: pgmap v69: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:53.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:53 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-12T22:58:53.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:53 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-12T22:58:53.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:53 smithi181 ceph-mon[26930]: osdmap e21: 3 total, 3 up, 3 in 2024-04-12T22:58:53.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:53 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-12T22:58:53.855 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 22:58:53 smithi144 sudo[44592]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T22:58:53.855 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 22:58:53 smithi144 sudo[44592]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:53.855 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 22:58:53 smithi144 sudo[44592]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:54.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T22:58:54.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T22:58:54.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:54.158 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:58:54.158 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:58:54.158 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:54.158 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:54.158 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44624]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T22:58:54.158 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44624]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:54.158 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44624]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: osdmap e22: 3 total, 3 up, 3 in 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:54.524 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44639]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T22:58:54.525 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44639]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:54.525 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44639]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:54.804 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44647]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T22:58:54.805 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44647]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:54.805 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44647]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:54.805 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44650]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T22:58:54.805 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44650]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:54.805 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:54 smithi144 sudo[44650]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:55.056 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44672]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T22:58:55.056 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44672]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:55.056 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44672]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:55.314 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 ceph-mon[24815]: from='client.14290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:58:55.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 ceph-mon[24815]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:55.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 ceph-mon[24815]: osdmap e23: 3 total, 3 up, 3 in 2024-04-12T22:58:55.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44681]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T22:58:55.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44681]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:55.315 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44681]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:55.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:55 smithi181 ceph-mon[26930]: from='client.14290 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi144:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:58:55.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:55 smithi181 ceph-mon[26930]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:55.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:55 smithi181 ceph-mon[26930]: osdmap e23: 3 total, 3 up, 3 in 2024-04-12T22:58:55.956 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44707]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T22:58:55.956 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44707]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:55.956 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:55 smithi144 sudo[44707]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:56.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:55 smithi181 sudo[29236]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-12T22:58:56.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:55 smithi181 sudo[29236]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:56.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:55 smithi181 sudo[29236]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: mgrmap e19: smithi144.rlueky(active, since 2m), standbys: smithi181.whsrmz 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:58:56.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:56 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 sudo[29239]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 sudo[29239]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 sudo[29239]: pam_unix(sudo:session): session closed for user root 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: mgrmap e19: smithi144.rlueky(active, since 2m), standbys: smithi181.whsrmz 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T22:58:56.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:56 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T22:58:57.211 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:57.212 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-12T22:58:57.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:58:57.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:57.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:58:57.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:58:57.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:58:57.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:58:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:58 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi144 to 4288M 2024-04-12T22:58:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:58 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3168340576' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d96d2ee5-baf4-4761-abbe-657573a477a2"}]: dispatch 2024-04-12T22:58:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:58 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3168340576' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d96d2ee5-baf4-4761-abbe-657573a477a2"}]': finished 2024-04-12T22:58:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:58 smithi181 ceph-mon[26930]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T22:58:58.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:58 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:58:58.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:58 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi144 to 4288M 2024-04-12T22:58:58.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:58 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3168340576' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d96d2ee5-baf4-4761-abbe-657573a477a2"}]: dispatch 2024-04-12T22:58:58.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:58 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3168340576' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d96d2ee5-baf4-4761-abbe-657573a477a2"}]': finished 2024-04-12T22:58:58.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:58 smithi144 ceph-mon[24815]: osdmap e24: 4 total, 3 up, 4 in 2024-04-12T22:58:58.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:58 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:58:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:59 smithi181 ceph-mon[26930]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:59.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:58:59 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1560381850' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:58:59.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:59 smithi144 ceph-mon[24815]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:58:59.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:58:59 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1560381850' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:59:01.299 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:01 smithi144 ceph-mon[24815]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:59:01.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:01 smithi181 ceph-mon[26930]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:59:03.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:03 smithi181 ceph-mon[26930]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-12T22:59:03.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:03 smithi144 ceph-mon[24815]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-12T22:59:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:04 smithi144 ceph-mon[24815]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-12T22:59:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:04 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T22:59:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:04 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:04 smithi144 ceph-mon[24815]: Deploying daemon osd.3 on smithi144 2024-04-12T22:59:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:04 smithi181 ceph-mon[26930]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-12T22:59:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:04 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T22:59:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:04 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:04 smithi181 ceph-mon[26930]: Deploying daemon osd.3 on smithi144 2024-04-12T22:59:06.985 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:06 smithi144 ceph-mon[24815]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T22:59:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:06 smithi181 ceph-mon[26930]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T22:59:08.995 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:08 smithi181 ceph-mon[26930]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T22:59:08.995 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:08.995 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:08.995 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:08.995 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:08 smithi144 ceph-mon[24815]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-12T22:59:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:11.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:10 smithi144 ceph-mon[24815]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-04-12T22:59:11.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:10 smithi181 ceph-mon[26930]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 66 KiB/s wr, 4 op/s 2024-04-12T22:59:11.407 INFO:teuthology.orchestra.run.smithi144.stdout:Created osd(s) 3 on host 'smithi144' 2024-04-12T22:59:12.323 DEBUG:teuthology.orchestra.run.smithi144:osd.3> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.3.service 2024-04-12T22:59:12.325 INFO:tasks.cephadm:Deploying osd.4 on smithi181 with /dev/vg_nvme/lv_4... 2024-04-12T22:59:12.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-12T22:59:12.559 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:12 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:12.560 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:12 smithi144 ceph-mon[24815]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-12T22:59:12.560 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:12 smithi144 ceph-mon[24815]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T22:59:12.639 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:12 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:12.639 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:12 smithi181 ceph-mon[26930]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-12T22:59:12.639 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:12 smithi181 ceph-mon[26930]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T22:59:13.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:13 smithi144 ceph-mon[24815]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T22:59:13.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:13 smithi144 ceph-mon[24815]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T22:59:13.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:13.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:13 smithi144 ceph-mon[24815]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:59:13.525 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 22:59:13 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[46279]: 2024-04-12T22:59:13.431+0000 7f3bc718a700 -1 osd.3 0 waiting for initial osdmap 2024-04-12T22:59:13.525 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 22:59:13 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[46279]: 2024-04-12T22:59:13.439+0000 7f3bc2322700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T22:59:13.688 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:13 smithi181 ceph-mon[26930]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T22:59:13.689 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:13 smithi181 ceph-mon[26930]: osdmap e25: 4 total, 3 up, 4 in 2024-04-12T22:59:13.689 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:13.689 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:13 smithi181 ceph-mon[26930]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T22:59:14.210 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-12T22:59:14.210 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-12T22:59:14.210 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-04-12T22:59:14.210 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-04-12T22:59:14.211 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117179 s, 895 MB/s 2024-04-12T22:59:14.211 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-04-12T22:59:14.474 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:14 smithi181 ceph-mon[26930]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:59:14.474 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:14 smithi181 ceph-mon[26930]: osdmap e26: 4 total, 3 up, 4 in 2024-04-12T22:59:14.474 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:14.474 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:14.474 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:14 smithi181 ceph-mon[26930]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:59:14.550 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:14 smithi144 ceph-mon[24815]: from='osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]': finished 2024-04-12T22:59:14.550 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:14 smithi144 ceph-mon[24815]: osdmap e26: 4 total, 3 up, 4 in 2024-04-12T22:59:14.550 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:14.550 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:14.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:14 smithi144 ceph-mon[24815]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-12T22:59:14.843 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_4 2024-04-12T22:59:15.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T22:59:15.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T22:59:15.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:15.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159] boot 2024-04-12T22:59:15.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T22:59:15.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:15.496 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:15 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: osd.3 [v2:172.21.15.144:6826/4132180159,v1:172.21.15.144:6827/4132180159] boot 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: osdmap e27: 4 total, 4 up, 4 in 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T22:59:15.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:15 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:16.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:16 smithi144 ceph-mon[24815]: osdmap e28: 4 total, 4 up, 4 in 2024-04-12T22:59:16.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:16 smithi144 ceph-mon[24815]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:16.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:16 smithi181 ceph-mon[26930]: osdmap e28: 4 total, 4 up, 4 in 2024-04-12T22:59:16.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:16 smithi181 ceph-mon[26930]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:17.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:17 smithi144 ceph-mon[24815]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:59:17.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:17 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:59:17.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:17 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:59:17.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:17 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:17.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:17 smithi181 ceph-mon[26930]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:59:17.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:17 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:59:17.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:17 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:59:17.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:17 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:18.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:18.526 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:18 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:18.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:18.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:18 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi144 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi144 to 3216M 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/2301417689' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "89ee3d1c-0f22-4e94-b850-9d0435cb5f12"}]: dispatch 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "89ee3d1c-0f22-4e94-b850-9d0435cb5f12"}]: dispatch 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "89ee3d1c-0f22-4e94-b850-9d0435cb5f12"}]': finished 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:19 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/1762852836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi144 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi144 to 3216M 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/2301417689' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "89ee3d1c-0f22-4e94-b850-9d0435cb5f12"}]: dispatch 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "89ee3d1c-0f22-4e94-b850-9d0435cb5f12"}]: dispatch 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "89ee3d1c-0f22-4e94-b850-9d0435cb5f12"}]': finished 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: osdmap e29: 5 total, 4 up, 5 in 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:20.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:19 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/1762852836' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:59:21.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:20 smithi144 ceph-mon[24815]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:21.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:20 smithi181 ceph-mon[26930]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:23.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:22 smithi144 ceph-mon[24815]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:23.280 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:22 smithi181 ceph-mon[26930]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:25.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:24 smithi144 ceph-mon[24815]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:25.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T22:59:25.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:25.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:24 smithi181 ceph-mon[26930]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:25.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T22:59:25.201 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:26.259 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:25 smithi181 ceph-mon[26930]: Deploying daemon osd.4 on smithi181 2024-04-12T22:59:26.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:25 smithi144 ceph-mon[24815]: Deploying daemon osd.4 on smithi181 2024-04-12T22:59:27.258 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:26 smithi181 ceph-mon[26930]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:27.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:26 smithi144 ceph-mon[24815]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:29.083 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:29 smithi181 ceph-mon[26930]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:29.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:29 smithi144 ceph-mon[24815]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:30.896 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:30.896 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:30.897 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:30.897 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:30.897 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:30 smithi181 ceph-mon[26930]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:31.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:31.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:31.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:31.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:31.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:30 smithi144 ceph-mon[24815]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:32.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:32 smithi181 ceph-mon[26930]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:32.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:32 smithi181 ceph-mon[26930]: from='osd.4 [v2:172.21.15.181:6800/1284091541,v1:172.21.15.181:6801/1284091541]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T22:59:32.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:32 smithi181 ceph-mon[26930]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T22:59:33.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:32 smithi144 ceph-mon[24815]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:33.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:32 smithi144 ceph-mon[24815]: from='osd.4 [v2:172.21.15.181:6800/1284091541,v1:172.21.15.181:6801/1284091541]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T22:59:33.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:32 smithi144 ceph-mon[24815]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T22:59:33.399 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 4 on host 'smithi181' 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='osd.4 [v2:172.21.15.181:6800/1284091541,v1:172.21.15.181:6801/1284091541]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:33.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:34.017 DEBUG:teuthology.orchestra.run.smithi181:osd.4> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.4.service 2024-04-12T22:59:34.019 INFO:tasks.cephadm:Deploying osd.5 on smithi181 with /dev/vg_nvme/lv_3... 2024-04-12T22:59:34.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: osdmap e30: 5 total, 4 up, 5 in 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='osd.4 [v2:172.21.15.181:6800/1284091541,v1:172.21.15.181:6801/1284091541]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:34.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:34 smithi144 ceph-mon[24815]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:34 smithi144 ceph-mon[24815]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T22:59:35.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:34 smithi144 ceph-mon[24815]: osdmap e31: 5 total, 4 up, 5 in 2024-04-12T22:59:35.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:35.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:35.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:34 smithi181 ceph-mon[26930]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:35.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:34 smithi181 ceph-mon[26930]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T22:59:35.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:34 smithi181 ceph-mon[26930]: osdmap e31: 5 total, 4 up, 5 in 2024-04-12T22:59:35.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:35.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:35.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:35 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T22:59:35.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:35 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T22:59:35.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:35 smithi181 ceph-mon[26930]: osd.4 [v2:172.21.15.181:6800/1284091541,v1:172.21.15.181:6801/1284091541] boot 2024-04-12T22:59:35.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:35 smithi181 ceph-mon[26930]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T22:59:35.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:35.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:35 smithi181 ceph-mon[26930]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T22:59:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:35 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T22:59:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:35 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T22:59:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:35 smithi144 ceph-mon[24815]: osd.4 [v2:172.21.15.181:6800/1284091541,v1:172.21.15.181:6801/1284091541] boot 2024-04-12T22:59:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:35 smithi144 ceph-mon[24815]: osdmap e32: 5 total, 5 up, 5 in 2024-04-12T22:59:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T22:59:36.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:35 smithi144 ceph-mon[24815]: osdmap e33: 5 total, 5 up, 5 in 2024-04-12T22:59:36.871 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:36 smithi181 ceph-mon[26930]: pgmap v103: 1 pgs: 1 remapped+peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:36.871 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:36.871 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:36.871 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:36.871 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:36.872 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:36 smithi181 ceph-mon[26930]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T22:59:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:36 smithi144 ceph-mon[24815]: pgmap v103: 1 pgs: 1 remapped+peering; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-12T22:59:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T22:59:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:37.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:36 smithi144 ceph-mon[24815]: osdmap e34: 5 total, 5 up, 5 in 2024-04-12T22:59:37.889 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-12T22:59:37.889 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-12T22:59:37.889 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-04-12T22:59:37.889 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-04-12T22:59:37.889 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112914 s, 929 MB/s 2024-04-12T22:59:37.889 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-04-12T22:59:38.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:37 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi181 2024-04-12T22:59:38.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:37 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi181 to 15936M 2024-04-12T22:59:38.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:37 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi181 2024-04-12T22:59:38.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:37 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi181 to 15936M 2024-04-12T22:59:38.605 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_3 2024-04-12T22:59:38.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:38 smithi181 ceph-mon[26930]: pgmap v106: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:38.916 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:38 smithi181 ceph-mon[26930]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T22:59:39.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:38 smithi144 ceph-mon[24815]: pgmap v106: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:39.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:38 smithi144 ceph-mon[24815]: osdmap e35: 5 total, 5 up, 5 in 2024-04-12T22:59:40.139 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:39 smithi181 ceph-mon[26930]: osdmap e36: 5 total, 5 up, 5 in 2024-04-12T22:59:40.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:39 smithi144 ceph-mon[24815]: osdmap e36: 5 total, 5 up, 5 in 2024-04-12T22:59:40.931 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:40 smithi181 ceph-mon[26930]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:40.931 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:40 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:59:40.931 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:40 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:59:40.931 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:40 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:40 smithi144 ceph-mon[24815]: pgmap v109: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:40 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T22:59:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:40 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T22:59:41.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:40 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:42.092 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:41 smithi181 ceph-mon[26930]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:59:42.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:41 smithi144 ceph-mon[24815]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T22:59:43.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:42 smithi181 ceph-mon[26930]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-12T22:59:43.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:42 smithi144 ceph-mon[24815]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-12T22:59:44.176 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:43 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/74918894' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133"}]: dispatch 2024-04-12T22:59:44.176 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:43 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133"}]: dispatch 2024-04-12T22:59:44.176 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:43 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133"}]': finished 2024-04-12T22:59:44.176 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:43 smithi144 ceph-mon[24815]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T22:59:44.177 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:43 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T22:59:44.177 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:43 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/1152004342' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:59:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:43 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/74918894' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133"}]: dispatch 2024-04-12T22:59:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:43 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133"}]: dispatch 2024-04-12T22:59:44.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:43 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133"}]': finished 2024-04-12T22:59:44.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:43 smithi181 ceph-mon[26930]: osdmap e37: 6 total, 5 up, 6 in 2024-04-12T22:59:44.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:43 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T22:59:44.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:43 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/1152004342' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T22:59:45.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:44 smithi144 ceph-mon[24815]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T22:59:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:44 smithi181 ceph-mon[26930]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T22:59:47.115 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:46 smithi181 ceph-mon[26930]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-12T22:59:47.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:46 smithi144 ceph-mon[24815]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-12T22:59:49.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:48 smithi181 ceph-mon[26930]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T22:59:49.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:48 smithi144 ceph-mon[24815]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-12T22:59:50.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T22:59:50.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:50.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:59:50.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:59:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T22:59:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T22:59:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T22:59:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:50 smithi181 ceph-mon[26930]: Deploying daemon osd.5 on smithi181 2024-04-12T22:59:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:50 smithi181 ceph-mon[26930]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T22:59:51.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:50 smithi144 ceph-mon[24815]: Deploying daemon osd.5 on smithi181 2024-04-12T22:59:51.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:50 smithi144 ceph-mon[24815]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T22:59:53.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:52 smithi144 ceph-mon[24815]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:53.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:52 smithi181 ceph-mon[26930]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:54 smithi144 ceph-mon[24815]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:55.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:54 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:55.026 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:54 smithi181 ceph-mon[26930]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:55.027 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:55.027 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:55.027 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:55.027 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:54 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:57.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:56 smithi181 ceph-mon[26930]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:57.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:56 smithi144 ceph-mon[24815]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:58.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:57 smithi181 ceph-mon[26930]: from='osd.5 [v2:172.21.15.181:6808/994147777,v1:172.21.15.181:6809/994147777]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T22:59:58.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:57 smithi181 ceph-mon[26930]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T22:59:58.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:58.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:57 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:58.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:57 smithi144 ceph-mon[24815]: from='osd.5 [v2:172.21.15.181:6808/994147777,v1:172.21.15.181:6809/994147777]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T22:59:58.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:57 smithi144 ceph-mon[24815]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T22:59:58.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:58.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:57 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:58.307 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 5 on host 'smithi181' 2024-04-12T22:59:58.970 DEBUG:teuthology.orchestra.run.smithi181:osd.5> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.5.service 2024-04-12T22:59:58.972 INFO:tasks.cephadm:Deploying osd.6 on smithi181 with /dev/vg_nvme/lv_2... 2024-04-12T22:59:58.973 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='osd.5 [v2:172.21.15.181:6808/994147777,v1:172.21.15.181:6809/994147777]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:59.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:58 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: osdmap e38: 6 total, 5 up, 6 in 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='osd.5 [v2:172.21.15.181:6808/994147777,v1:172.21.15.181:6809/994147777]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T22:59:59.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:58 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:00.188 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:59 smithi181 ceph-mon[26930]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T23:00:00.188 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:59 smithi181 ceph-mon[26930]: osdmap e39: 6 total, 5 up, 6 in 2024-04-12T23:00:00.188 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:00:00.189 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 22:59:59 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:00:00.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:59 smithi144 ceph-mon[24815]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T23:00:00.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:59 smithi144 ceph-mon[24815]: osdmap e39: 6 total, 5 up, 6 in 2024-04-12T23:00:00.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:00:00.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 22:59:59 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:00:01.077 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: pgmap v122: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: osd.5 [v2:172.21.15.181:6808/994147777,v1:172.21.15.181:6809/994147777] boot 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: Health detail: HEALTH_WARN Reduced data availability: 1 pg peering 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg peering 2024-04-12T23:00:01.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:00 smithi181 ceph-mon[26930]: pg 1.0 is stuck peering for 63s, current state peering, last acting [0,1] 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: pgmap v122: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: osd.5 [v2:172.21.15.181:6808/994147777,v1:172.21.15.181:6809/994147777] boot 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: osdmap e40: 6 total, 6 up, 6 in 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: Health detail: HEALTH_WARN Reduced data availability: 1 pg peering 2024-04-12T23:00:01.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: [WRN] PG_AVAILABILITY: Reduced data availability: 1 pg peering 2024-04-12T23:00:01.276 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:00 smithi144 ceph-mon[24815]: pg 1.0 is stuck peering for 63s, current state peering, last acting [0,1] 2024-04-12T23:00:01.925 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-12T23:00:01.925 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-12T23:00:01.925 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-04-12T23:00:01.925 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-04-12T23:00:01.925 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012192 s, 860 MB/s 2024-04-12T23:00:01.925 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-04-12T23:00:02.194 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:02 smithi181 ceph-mon[26930]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T23:00:02.194 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:02 smithi181 ceph-mon[26930]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T23:00:02.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:01 smithi144 ceph-mon[24815]: osdmap e41: 6 total, 6 up, 6 in 2024-04-12T23:00:02.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:01 smithi144 ceph-mon[24815]: osdmap e42: 6 total, 6 up, 6 in 2024-04-12T23:00:02.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_2 2024-04-12T23:00:03.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:03 smithi181 ceph-mon[26930]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:03.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:03 smithi144 ceph-mon[24815]: pgmap v125: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:04.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:04 smithi144 ceph-mon[24815]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T23:00:04.440 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:04 smithi181 ceph-mon[26930]: osdmap e43: 6 total, 6 up, 6 in 2024-04-12T23:00:05.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: osdmap e44: 6 total, 6 up, 6 in 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:05 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: osdmap e44: 6 total, 6 up, 6 in 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:05.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:05 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:06 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi181 2024-04-12T23:00:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:06 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi181 to 7968M 2024-04-12T23:00:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:06 smithi181 ceph-mon[26930]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:00:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:06 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T23:00:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:06 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T23:00:06.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:06 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:06.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:06 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi181 2024-04-12T23:00:06.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:06 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi181 to 7968M 2024-04-12T23:00:06.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:06 smithi144 ceph-mon[24815]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:00:06.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:06 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T23:00:06.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:06 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T23:00:06.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:06 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:07.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:07 smithi181 ceph-mon[26930]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-04-12T23:00:07.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:07 smithi181 ceph-mon[26930]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T23:00:07.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:07 smithi181 ceph-mon[26930]: Cluster is now healthy 2024-04-12T23:00:07.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:07 smithi144 ceph-mon[24815]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-04-12T23:00:07.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:07 smithi144 ceph-mon[24815]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-12T23:00:07.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:07 smithi144 ceph-mon[24815]: Cluster is now healthy 2024-04-12T23:00:08.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:08 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/3184323488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87d9330e-dc50-47bd-9767-bb34482df92f"}]: dispatch 2024-04-12T23:00:08.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:08 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87d9330e-dc50-47bd-9767-bb34482df92f"}]: dispatch 2024-04-12T23:00:08.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:08 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87d9330e-dc50-47bd-9767-bb34482df92f"}]': finished 2024-04-12T23:00:08.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:08 smithi144 ceph-mon[24815]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T23:00:08.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:08.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:08 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/3184323488' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87d9330e-dc50-47bd-9767-bb34482df92f"}]: dispatch 2024-04-12T23:00:08.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:08 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "87d9330e-dc50-47bd-9767-bb34482df92f"}]: dispatch 2024-04-12T23:00:08.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:08 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "87d9330e-dc50-47bd-9767-bb34482df92f"}]': finished 2024-04-12T23:00:08.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:08 smithi181 ceph-mon[26930]: osdmap e45: 7 total, 6 up, 7 in 2024-04-12T23:00:08.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:09.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:09 smithi144 ceph-mon[24815]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-12T23:00:09.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:09 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/3395297123' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T23:00:09.349 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:09 smithi181 ceph-mon[26930]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-12T23:00:09.349 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:09 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/3395297123' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T23:00:11.089 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:11 smithi181 ceph-mon[26930]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-12T23:00:11.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:11 smithi144 ceph-mon[24815]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-04-12T23:00:13.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:13 smithi181 ceph-mon[26930]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-12T23:00:13.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:13 smithi144 ceph-mon[24815]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-12T23:00:14.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:14 smithi181 ceph-mon[26930]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-12T23:00:14.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T23:00:14.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:14 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:14.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:14 smithi181 ceph-mon[26930]: Deploying daemon osd.6 on smithi181 2024-04-12T23:00:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:14 smithi144 ceph-mon[24815]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-04-12T23:00:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T23:00:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:14 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:15.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:14 smithi144 ceph-mon[24815]: Deploying daemon osd.6 on smithi181 2024-04-12T23:00:17.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:16 smithi144 ceph-mon[24815]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:17.139 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:16 smithi181 ceph-mon[26930]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:19.000 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:18 smithi181 ceph-mon[26930]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:19.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:18 smithi144 ceph-mon[24815]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:20.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:20.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:20.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:20.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:20.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:20.326 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:20.326 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:20.326 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:20.326 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:20.326 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:21 smithi181 ceph-mon[26930]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:21.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:21 smithi144 ceph-mon[24815]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:22.139 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 6 on host 'smithi181' 2024-04-12T23:00:22.754 DEBUG:teuthology.orchestra.run.smithi181:osd.6> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.6.service 2024-04-12T23:00:22.756 INFO:tasks.cephadm:Deploying osd.7 on smithi181 with /dev/vg_nvme/lv_1... 2024-04-12T23:00:22.756 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-12T23:00:23.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:23 smithi144 ceph-mon[24815]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:23.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:23 smithi144 ceph-mon[24815]: from='osd.6 [v2:172.21.15.181:6816/2565681235,v1:172.21.15.181:6817/2565681235]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:00:23.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:23 smithi144 ceph-mon[24815]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:00:23.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:23.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:23 smithi181 ceph-mon[26930]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:23.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:23 smithi181 ceph-mon[26930]: from='osd.6 [v2:172.21.15.181:6816/2565681235,v1:172.21.15.181:6817/2565681235]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:00:23.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:23 smithi181 ceph-mon[26930]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:00:23.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:24.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-mon[26930]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T23:00:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-mon[26930]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T23:00:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-mon[26930]: from='osd.6 [v2:172.21.15.181:6816/2565681235,v1:172.21.15.181:6817/2565681235]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-mon[26930]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:24.199 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[37095]: 2024-04-12T23:00:24.042+0000 7f4d758bb700 -1 osd.6 0 waiting for initial osdmap 2024-04-12T23:00:24.199 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:00:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[37095]: 2024-04-12T23:00:24.051+0000 7f4d72256700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:00:24.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:24 smithi144 ceph-mon[24815]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T23:00:24.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:24 smithi144 ceph-mon[24815]: osdmap e46: 7 total, 6 up, 7 in 2024-04-12T23:00:24.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:24.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:24 smithi144 ceph-mon[24815]: from='osd.6 [v2:172.21.15.181:6816/2565681235,v1:172.21.15.181:6817/2565681235]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:24.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:24 smithi144 ceph-mon[24815]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:25.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:25 smithi181 ceph-mon[26930]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:25.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:25 smithi181 ceph-mon[26930]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T23:00:25.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:25 smithi181 ceph-mon[26930]: osdmap e47: 7 total, 6 up, 7 in 2024-04-12T23:00:25.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:25 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:25.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:25 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:25.260 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-12T23:00:25.260 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-12T23:00:25.261 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-04-12T23:00:25.261 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-04-12T23:00:25.261 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124239 s, 844 MB/s 2024-04-12T23:00:25.261 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-04-12T23:00:25.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:25 smithi144 ceph-mon[24815]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:25.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:25 smithi144 ceph-mon[24815]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T23:00:25.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:25 smithi144 ceph-mon[24815]: osdmap e47: 7 total, 6 up, 7 in 2024-04-12T23:00:25.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:25 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:25.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:25 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:25.864 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi181:vg_nvme/lv_1 2024-04-12T23:00:26.123 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T23:00:26.123 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T23:00:26.123 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: osd.6 [v2:172.21.15.181:6816/2565681235,v1:172.21.15.181:6817/2565681235] boot 2024-04-12T23:00:26.124 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T23:00:26.124 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:26.124 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:26.124 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:26 smithi181 ceph-mon[26930]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: osd.6 [v2:172.21.15.181:6816/2565681235,v1:172.21.15.181:6817/2565681235] boot 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: osdmap e48: 7 total, 7 up, 7 in 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:26.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:26 smithi144 ceph-mon[24815]: osdmap e49: 7 total, 7 up, 7 in 2024-04-12T23:00:27.417 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:27 smithi181 ceph-mon[26930]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:27.417 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:27 smithi181 ceph-mon[26930]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T23:00:27.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:27 smithi144 ceph-mon[24815]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-12T23:00:27.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:27 smithi144 ceph-mon[24815]: osdmap e50: 7 total, 7 up, 7 in 2024-04-12T23:00:29.323 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:29 smithi181 ceph-mon[26930]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:29.323 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:29 smithi181 ceph-mon[26930]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T23:00:29.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:29 smithi144 ceph-mon[24815]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:29.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:29 smithi144 ceph-mon[24815]: osdmap e51: 7 total, 7 up, 7 in 2024-04-12T23:00:30.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: osdmap e52: 7 total, 7 up, 7 in 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:30.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: osdmap e52: 7 total, 7 up, 7 in 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:30.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:31.237 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:31 smithi181 ceph-mon[26930]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:00:31.237 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:31 smithi181 ceph-mon[26930]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:31.237 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:31 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi181 2024-04-12T23:00:31.237 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:31 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi181 to 5312M 2024-04-12T23:00:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:31 smithi144 ceph-mon[24815]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:00:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:31 smithi144 ceph-mon[24815]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:31 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi181 2024-04-12T23:00:31.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:31 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi181 to 5312M 2024-04-12T23:00:32.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:32 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/4285772706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "83c589a3-40ab-417e-9109-02ca5bfee57c"}]: dispatch 2024-04-12T23:00:32.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:32 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "83c589a3-40ab-417e-9109-02ca5bfee57c"}]: dispatch 2024-04-12T23:00:32.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:32 smithi181 ceph-mon[26930]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "83c589a3-40ab-417e-9109-02ca5bfee57c"}]': finished 2024-04-12T23:00:32.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:32 smithi181 ceph-mon[26930]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T23:00:32.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:32 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:32.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:32 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/4285772706' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "83c589a3-40ab-417e-9109-02ca5bfee57c"}]: dispatch 2024-04-12T23:00:32.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:32 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "83c589a3-40ab-417e-9109-02ca5bfee57c"}]: dispatch 2024-04-12T23:00:32.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:32 smithi144 ceph-mon[24815]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "83c589a3-40ab-417e-9109-02ca5bfee57c"}]': finished 2024-04-12T23:00:32.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:32 smithi144 ceph-mon[24815]: osdmap e53: 8 total, 7 up, 8 in 2024-04-12T23:00:32.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:32 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:33.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:33 smithi181 ceph-mon[26930]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-12T23:00:33.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:33 smithi181 ceph-mon[26930]: from='client.? 172.21.15.181:0/1428322975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T23:00:33.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:33 smithi144 ceph-mon[24815]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-12T23:00:33.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:33 smithi144 ceph-mon[24815]: from='client.? 172.21.15.181:0/1428322975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-12T23:00:35.319 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:35 smithi181 ceph-mon[26930]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T23:00:35.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:35 smithi144 ceph-mon[24815]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T23:00:37.369 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:37 smithi181 ceph-mon[26930]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-12T23:00:37.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:37 smithi144 ceph-mon[24815]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-04-12T23:00:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T23:00:38.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:38.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T23:00:38.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:39.395 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:39 smithi181 ceph-mon[26930]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-12T23:00:39.396 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:39 smithi181 ceph-mon[26930]: Deploying daemon osd.7 on smithi181 2024-04-12T23:00:39.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:39 smithi144 ceph-mon[24815]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-12T23:00:39.525 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:39 smithi144 ceph-mon[24815]: Deploying daemon osd.7 on smithi181 2024-04-12T23:00:41.337 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:41 smithi181 ceph-mon[26930]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T23:00:41.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:41 smithi144 ceph-mon[24815]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T23:00:43.249 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:43 smithi181 ceph-mon[26930]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:43.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:43 smithi144 ceph-mon[24815]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:44.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:44 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:44.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:44 smithi181 ceph-mon[26930]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:44.760 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:44.761 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:44.761 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:44.761 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:44 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:44.761 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:44 smithi144 ceph-mon[24815]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:46.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:46 smithi181 ceph-mon[26930]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:46.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:46 smithi181 ceph-mon[26930]: from='osd.7 [v2:172.21.15.181:6824/1826485044,v1:172.21.15.181:6825/1826485044]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:00:46.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:46 smithi181 ceph-mon[26930]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:00:47.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:46 smithi144 ceph-mon[24815]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:47.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:46 smithi144 ceph-mon[24815]: from='osd.7 [v2:172.21.15.181:6824/1826485044,v1:172.21.15.181:6825/1826485044]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:00:47.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:46 smithi144 ceph-mon[24815]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:00:47.292 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 7 on host 'smithi181' 2024-04-12T23:00:47.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T23:00:47.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T23:00:47.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:47.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: from='osd.7 [v2:172.21.15.181:6824/1826485044,v1:172.21.15.181:6825/1826485044]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:47.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:47.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:47.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:48.033 DEBUG:teuthology.orchestra.run.smithi181:osd.7> sudo journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.7.service 2024-04-12T23:00:48.035 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-12T23:00:48.036 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd stat -f json 2024-04-12T23:00:48.064 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T23:00:48.064 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: osdmap e54: 8 total, 7 up, 8 in 2024-04-12T23:00:48.064 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:48.064 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: from='osd.7 [v2:172.21.15.181:6824/1826485044,v1:172.21.15.181:6825/1826485044]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:48.065 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:00:48.065 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:48.065 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: osdmap e55: 8 total, 7 up, 8 in 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: osd.7 [v2:172.21.15.181:6824/1826485044,v1:172.21.15.181:6825/1826485044] boot 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T23:00:49.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:48 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: osdmap e55: 8 total, 7 up, 8 in 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: osd.7 [v2:172.21.15.181:6824/1826485044,v1:172.21.15.181:6825/1826485044] boot 2024-04-12T23:00:49.125 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: osdmap e56: 8 total, 8 up, 8 in 2024-04-12T23:00:49.126 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:48 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:00:49.903 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:00:50.161 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:49 smithi181 ceph-mon[26930]: purged_snaps scrub starts 2024-04-12T23:00:50.161 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:49 smithi181 ceph-mon[26930]: purged_snaps scrub ok 2024-04-12T23:00:50.161 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:49 smithi181 ceph-mon[26930]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T23:00:50.161 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:00:50.161 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:00:50.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:49 smithi144 ceph-mon[24815]: purged_snaps scrub starts 2024-04-12T23:00:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:49 smithi144 ceph-mon[24815]: purged_snaps scrub ok 2024-04-12T23:00:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:49 smithi144 ceph-mon[24815]: osdmap e57: 8 total, 8 up, 8 in 2024-04-12T23:00:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:00:50.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:00:50.551 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1712962848,"num_in_osds":8,"osd_in_since":1712962831,"num_remapped_pgs":1} 2024-04-12T23:00:50.551 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-12T23:00:50.822 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:00:50.944 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: pgmap v164: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2429338398' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:50.945 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: pgmap v164: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2429338398' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: osdmap e58: 8 total, 8 up, 8 in 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:00:51.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:52.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: Detected new or changed devices on smithi181 2024-04-12T23:00:52.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi181 to 3984M 2024-04-12T23:00:52.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:52.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:52.015 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:52.015 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:52.015 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:52.015 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:51 smithi144 ceph-mon[24815]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T23:00:52.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: Detected new or changed devices on smithi181 2024-04-12T23:00:52.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi181 to 3984M 2024-04-12T23:00:52.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:52.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:00:52.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:00:52.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:00:52.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:00:52.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:51 smithi181 ceph-mon[26930]: osdmap e59: 8 total, 8 up, 8 in 2024-04-12T23:00:52.396 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:00:52.396 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":59,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","created":"2024-04-12T22:54:54.145779+0000","modified":"2024-04-12T23:00:51.751861+0000","last_up_change":"2024-04-12T23:00:48.748201+0000","last_in_change":"2024-04-12T23:00:31.912858+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T22:58:52.007841+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"dd4c61f1-5d94-4c3b-87e1-2486e2858ba7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6803","nonce":4186511534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6805","nonce":4186511534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6809","nonce":4186511534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6807","nonce":4186511534}]},"public_addr":"172.21.15.144:6803/4186511534","cluster_addr":"172.21.15.144:6805/4186511534","heartbeat_back_addr":"172.21.15.144:6809/4186511534","heartbeat_front_addr":"172.21.15.144:6807/4186511534","state":["exists","up"]},{"osd":1,"uuid":"115fcd46-373b-42b7-9598-a226a86ac8ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6811","nonce":73710096}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6813","nonce":73710096}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6817","nonce":73710096}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6815","nonce":73710096}]},"public_addr":"172.21.15.144:6811/73710096","cluster_addr":"172.21.15.144:6813/73710096","heartbeat_back_addr":"172.21.15.144:6817/73710096","heartbeat_front_addr":"172.21.15.144:6815/73710096","state":["exists","up"]},{"osd":2,"uuid":"293a464f-f039-4f0f-8158-1111cf1760df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6819","nonce":961664561}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6821","nonce":961664561}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6825","nonce":961664561}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6823","nonce":961664561}]},"public_addr":"172.21.15.144:6819/961664561","cluster_addr":"172.21.15.144:6821/961664561","heartbeat_back_addr":"172.21.15.144:6825/961664561","heartbeat_front_addr":"172.21.15.144:6823/961664561","state":["exists","up"]},{"osd":3,"uuid":"d96d2ee5-baf4-4761-abbe-657573a477a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6827","nonce":4132180159}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6829","nonce":4132180159}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6832","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6833","nonce":4132180159}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6831","nonce":4132180159}]},"public_addr":"172.21.15.144:6827/4132180159","cluster_addr":"172.21.15.144:6829/4132180159","heartbeat_back_addr":"172.21.15.144:6833/4132180159","heartbeat_front_addr":"172.21.15.144:6831/4132180159","state":["exists","up"]},{"osd":4,"uuid":"89ee3d1c-0f22-4e94-b850-9d0435cb5f12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6801","nonce":1284091541}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6803","nonce":1284091541}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6807","nonce":1284091541}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6805","nonce":1284091541}]},"public_addr":"172.21.15.181:6801/1284091541","cluster_addr":"172.21.15.181:6803/1284091541","heartbeat_back_addr":"172.21.15.181:6807/1284091541","heartbeat_front_addr":"172.21.15.181:6805/1284091541","state":["exists","up"]},{"osd":5,"uuid":"2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6809","nonce":994147777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6811","nonce":994147777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6815","nonce":994147777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6813","nonce":994147777}]},"public_addr":"172.21.15.181:6809/994147777","cluster_addr":"172.21.15.181:6811/994147777","heartbeat_back_addr":"172.21.15.181:6815/994147777","heartbeat_front_addr":"172.21.15.181:6813/994147777","state":["exists","up"]},{"osd":6,"uuid":"87d9330e-dc50-47bd-9767-bb34482df92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6817","nonce":2565681235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6819","nonce":2565681235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6823","nonce":2565681235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6821","nonce":2565681235}]},"public_addr":"172.21.15.181:6817/2565681235","cluster_addr":"172.21.15.181:6819/2565681235","heartbeat_back_addr":"172.21.15.181:6823/2565681235","heartbeat_front_addr":"172.21.15.181:6821/2565681235","state":["exists","up"]},{"osd":7,"uuid":"83c589a3-40ab-417e-9109-02ca5bfee57c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6825","nonce":1826485044}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6827","nonce":1826485044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6831","nonce":1826485044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6829","nonce":1826485044}]},"public_addr":"172.21.15.181:6825/1826485044","cluster_addr":"172.21.15.181:6827/1826485044","heartbeat_back_addr":"172.21.15.181:6831/1826485044","heartbeat_front_addr":"172.21.15.181:6829/1826485044","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:57:52.420825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:58:23.343419+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:58:48.394844+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:12.771998+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:33.704827+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:58.445248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T23:00:23.116899+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T23:00:47.489482+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.144:6801/1971060878":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/1644164096":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/289197444":"2024-04-13T22:55:58.467573+0000","172.21.15.144:0/2748859989":"2024-04-13T22:55:58.467573+0000","172.21.15.144:0/1195888470":"2024-04-13T22:56:49.008974+0000","172.21.15.144:6801/4221422342":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6800/1971060878":"2024-04-13T22:56:49.008974+0000","172.21.15.144:6800/4221422342":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6800/2478980714":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/568903382":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/3814104170":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/824159127":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/1595559936":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6801/2478980714":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/1147693527":"2024-04-13T22:55:28.720269+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T23:00:53.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:52 smithi144 ceph-mon[24815]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:00:53.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:52 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/871165321' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T23:00:53.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:52 smithi144 ceph-mon[24815]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T23:00:53.083 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-12T22:58:52.007841+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-12T23:00:53.084 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-12T23:00:53.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:52 smithi181 ceph-mon[26930]: pgmap v167: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:00:53.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:52 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/871165321' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T23:00:53.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:52 smithi181 ceph-mon[26930]: osdmap e60: 8 total, 8 up, 8 in 2024-04-12T23:00:53.298 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:00:54.969 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:54 smithi144 ceph-mon[24815]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:00:55.168 INFO:teuthology.orchestra.run.smithi144.stdout:pg_num: 1 2024-04-12T23:00:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:54 smithi181 ceph-mon[26930]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:00:55.835 INFO:tasks.cephadm:Setting up client nodes... 2024-04-12T23:00:55.835 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-12T23:00:56.133 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:55 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1692651579' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T23:00:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:55 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1692651579' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-12T23:00:57.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:56 smithi144 ceph-mon[24815]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T23:00:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:56 smithi181 ceph-mon[26930]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-12T23:00:57.762 INFO:teuthology.orchestra.run.smithi144.stdout:[client.0] 2024-04-12T23:00:57.762 INFO:teuthology.orchestra.run.smithi144.stdout: key = AQApvRlm7vPvLRAA/RVxIDsjgW0LFWZVjAodnw== 2024-04-12T23:00:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:57 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1575553590' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T23:00:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:57 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1575553590' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T23:00:58.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:57 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1575553590' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-12T23:00:58.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:57 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1575553590' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-12T23:00:58.344 DEBUG:teuthology.orchestra.run.smithi144:> set -ex 2024-04-12T23:00:58.345 DEBUG:teuthology.orchestra.run.smithi144:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-12T23:00:58.345 DEBUG:teuthology.orchestra.run.smithi144:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-12T23:00:58.387 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-12T23:00:58.387 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-12T23:00:58.387 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph mgr dump --format=json 2024-04-12T23:00:58.638 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:00:59.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:00:58 smithi144 ceph-mon[24815]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-12T23:00:59.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:00:58 smithi181 ceph-mon[26930]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-12T23:01:00.467 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:01:01.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:00 smithi144 ceph-mon[24815]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-12T23:01:01.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:00 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3050744720' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T23:01:01.091 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:00 smithi181 ceph-mon[26930]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-12T23:01:01.091 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:00 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3050744720' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-12T23:01:01.095 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi144.rlueky","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6800","nonce":4240238233},{"type":"v1","addr":"172.21.15.144:6801","nonce":4240238233}]},"active_addr":"172.21.15.144:6801/4240238233","active_change":"2024-04-12T22:56:49.009381+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi181.whsrmz","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.144:8443/","prometheus":"http://172.21.15.144:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.144:0","nonce":1717434997}]},{"addrvec":[{"type":"v2","addr":"172.21.15.144:0","nonce":3870249318}]},{"addrvec":[{"type":"v2","addr":"172.21.15.144:0","nonce":1314343858}]},{"addrvec":[{"type":"v2","addr":"172.21.15.144:0","nonce":1126737368}]}]}} 2024-04-12T23:01:01.101 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-12T23:01:01.101 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-12T23:01:01.102 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-12T23:01:01.320 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:03.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:02 smithi144 ceph-mon[24815]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T23:01:03.049 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:01:03.049 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":60,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","created":"2024-04-12T22:54:54.145779+0000","modified":"2024-04-12T23:00:52.753274+0000","last_up_change":"2024-04-12T23:00:48.748201+0000","last_in_change":"2024-04-12T23:00:31.912858+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T22:58:52.007841+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"dd4c61f1-5d94-4c3b-87e1-2486e2858ba7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6803","nonce":4186511534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6805","nonce":4186511534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6809","nonce":4186511534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6807","nonce":4186511534}]},"public_addr":"172.21.15.144:6803/4186511534","cluster_addr":"172.21.15.144:6805/4186511534","heartbeat_back_addr":"172.21.15.144:6809/4186511534","heartbeat_front_addr":"172.21.15.144:6807/4186511534","state":["exists","up"]},{"osd":1,"uuid":"115fcd46-373b-42b7-9598-a226a86ac8ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6811","nonce":73710096}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6813","nonce":73710096}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6817","nonce":73710096}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6815","nonce":73710096}]},"public_addr":"172.21.15.144:6811/73710096","cluster_addr":"172.21.15.144:6813/73710096","heartbeat_back_addr":"172.21.15.144:6817/73710096","heartbeat_front_addr":"172.21.15.144:6815/73710096","state":["exists","up"]},{"osd":2,"uuid":"293a464f-f039-4f0f-8158-1111cf1760df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6819","nonce":961664561}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6821","nonce":961664561}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6825","nonce":961664561}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6823","nonce":961664561}]},"public_addr":"172.21.15.144:6819/961664561","cluster_addr":"172.21.15.144:6821/961664561","heartbeat_back_addr":"172.21.15.144:6825/961664561","heartbeat_front_addr":"172.21.15.144:6823/961664561","state":["exists","up"]},{"osd":3,"uuid":"d96d2ee5-baf4-4761-abbe-657573a477a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6827","nonce":4132180159}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6829","nonce":4132180159}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6832","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6833","nonce":4132180159}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6831","nonce":4132180159}]},"public_addr":"172.21.15.144:6827/4132180159","cluster_addr":"172.21.15.144:6829/4132180159","heartbeat_back_addr":"172.21.15.144:6833/4132180159","heartbeat_front_addr":"172.21.15.144:6831/4132180159","state":["exists","up"]},{"osd":4,"uuid":"89ee3d1c-0f22-4e94-b850-9d0435cb5f12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6801","nonce":1284091541}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6803","nonce":1284091541}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6807","nonce":1284091541}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6805","nonce":1284091541}]},"public_addr":"172.21.15.181:6801/1284091541","cluster_addr":"172.21.15.181:6803/1284091541","heartbeat_back_addr":"172.21.15.181:6807/1284091541","heartbeat_front_addr":"172.21.15.181:6805/1284091541","state":["exists","up"]},{"osd":5,"uuid":"2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6809","nonce":994147777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6811","nonce":994147777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6815","nonce":994147777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6813","nonce":994147777}]},"public_addr":"172.21.15.181:6809/994147777","cluster_addr":"172.21.15.181:6811/994147777","heartbeat_back_addr":"172.21.15.181:6815/994147777","heartbeat_front_addr":"172.21.15.181:6813/994147777","state":["exists","up"]},{"osd":6,"uuid":"87d9330e-dc50-47bd-9767-bb34482df92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6817","nonce":2565681235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6819","nonce":2565681235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6823","nonce":2565681235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6821","nonce":2565681235}]},"public_addr":"172.21.15.181:6817/2565681235","cluster_addr":"172.21.15.181:6819/2565681235","heartbeat_back_addr":"172.21.15.181:6823/2565681235","heartbeat_front_addr":"172.21.15.181:6821/2565681235","state":["exists","up"]},{"osd":7,"uuid":"83c589a3-40ab-417e-9109-02ca5bfee57c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6825","nonce":1826485044}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6827","nonce":1826485044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6831","nonce":1826485044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6829","nonce":1826485044}]},"public_addr":"172.21.15.181:6825/1826485044","cluster_addr":"172.21.15.181:6827/1826485044","heartbeat_back_addr":"172.21.15.181:6831/1826485044","heartbeat_front_addr":"172.21.15.181:6829/1826485044","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:57:52.420825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:58:23.343419+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:58:48.394844+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:12.771998+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:33.704827+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:58.445248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T23:00:23.116899+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T23:00:47.489482+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.144:6801/1971060878":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/1644164096":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/289197444":"2024-04-13T22:55:58.467573+0000","172.21.15.144:0/2748859989":"2024-04-13T22:55:58.467573+0000","172.21.15.144:0/1195888470":"2024-04-13T22:56:49.008974+0000","172.21.15.144:6801/4221422342":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6800/1971060878":"2024-04-13T22:56:49.008974+0000","172.21.15.144:6800/4221422342":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6800/2478980714":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/568903382":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/3814104170":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/824159127":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/1595559936":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6801/2478980714":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/1147693527":"2024-04-13T22:55:28.720269+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T23:01:03.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:02 smithi181 ceph-mon[26930]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-12T23:01:03.775 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-12T23:01:03.776 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-12T23:01:03.994 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:04.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:03 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3978272249' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T23:01:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:03 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3978272249' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T23:01:05.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:04 smithi144 ceph-mon[24815]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-12T23:01:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:04 smithi181 ceph-mon[26930]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-04-12T23:01:05.779 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:01:05.779 INFO:teuthology.orchestra.run.smithi144.stdout:{"epoch":60,"fsid":"9965523c-f91f-11ee-bc8d-c7b262605968","created":"2024-04-12T22:54:54.145779+0000","modified":"2024-04-12T23:00:52.753274+0000","last_up_change":"2024-04-12T23:00:48.748201+0000","last_in_change":"2024-04-12T23:00:31.912858+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-12T22:58:52.007841+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"dd4c61f1-5d94-4c3b-87e1-2486e2858ba7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6802","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6803","nonce":4186511534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6804","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6805","nonce":4186511534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6808","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6809","nonce":4186511534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6806","nonce":4186511534},{"type":"v1","addr":"172.21.15.144:6807","nonce":4186511534}]},"public_addr":"172.21.15.144:6803/4186511534","cluster_addr":"172.21.15.144:6805/4186511534","heartbeat_back_addr":"172.21.15.144:6809/4186511534","heartbeat_front_addr":"172.21.15.144:6807/4186511534","state":["exists","up"]},{"osd":1,"uuid":"115fcd46-373b-42b7-9598-a226a86ac8ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6810","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6811","nonce":73710096}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6812","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6813","nonce":73710096}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6816","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6817","nonce":73710096}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6814","nonce":73710096},{"type":"v1","addr":"172.21.15.144:6815","nonce":73710096}]},"public_addr":"172.21.15.144:6811/73710096","cluster_addr":"172.21.15.144:6813/73710096","heartbeat_back_addr":"172.21.15.144:6817/73710096","heartbeat_front_addr":"172.21.15.144:6815/73710096","state":["exists","up"]},{"osd":2,"uuid":"293a464f-f039-4f0f-8158-1111cf1760df","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6818","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6819","nonce":961664561}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6820","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6821","nonce":961664561}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6824","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6825","nonce":961664561}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6822","nonce":961664561},{"type":"v1","addr":"172.21.15.144:6823","nonce":961664561}]},"public_addr":"172.21.15.144:6819/961664561","cluster_addr":"172.21.15.144:6821/961664561","heartbeat_back_addr":"172.21.15.144:6825/961664561","heartbeat_front_addr":"172.21.15.144:6823/961664561","state":["exists","up"]},{"osd":3,"uuid":"d96d2ee5-baf4-4761-abbe-657573a477a2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6826","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6827","nonce":4132180159}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6828","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6829","nonce":4132180159}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6832","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6833","nonce":4132180159}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.144:6830","nonce":4132180159},{"type":"v1","addr":"172.21.15.144:6831","nonce":4132180159}]},"public_addr":"172.21.15.144:6827/4132180159","cluster_addr":"172.21.15.144:6829/4132180159","heartbeat_back_addr":"172.21.15.144:6833/4132180159","heartbeat_front_addr":"172.21.15.144:6831/4132180159","state":["exists","up"]},{"osd":4,"uuid":"89ee3d1c-0f22-4e94-b850-9d0435cb5f12","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6801","nonce":1284091541}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6803","nonce":1284091541}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6807","nonce":1284091541}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":1284091541},{"type":"v1","addr":"172.21.15.181:6805","nonce":1284091541}]},"public_addr":"172.21.15.181:6801/1284091541","cluster_addr":"172.21.15.181:6803/1284091541","heartbeat_back_addr":"172.21.15.181:6807/1284091541","heartbeat_front_addr":"172.21.15.181:6805/1284091541","state":["exists","up"]},{"osd":5,"uuid":"2721aa7f-5aa0-4d29-a63b-6e2fa8a1c133","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6809","nonce":994147777}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6811","nonce":994147777}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6815","nonce":994147777}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":994147777},{"type":"v1","addr":"172.21.15.181:6813","nonce":994147777}]},"public_addr":"172.21.15.181:6809/994147777","cluster_addr":"172.21.15.181:6811/994147777","heartbeat_back_addr":"172.21.15.181:6815/994147777","heartbeat_front_addr":"172.21.15.181:6813/994147777","state":["exists","up"]},{"osd":6,"uuid":"87d9330e-dc50-47bd-9767-bb34482df92f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6817","nonce":2565681235}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6819","nonce":2565681235}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6823","nonce":2565681235}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":2565681235},{"type":"v1","addr":"172.21.15.181:6821","nonce":2565681235}]},"public_addr":"172.21.15.181:6817/2565681235","cluster_addr":"172.21.15.181:6819/2565681235","heartbeat_back_addr":"172.21.15.181:6823/2565681235","heartbeat_front_addr":"172.21.15.181:6821/2565681235","state":["exists","up"]},{"osd":7,"uuid":"83c589a3-40ab-417e-9109-02ca5bfee57c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6825","nonce":1826485044}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6827","nonce":1826485044}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6831","nonce":1826485044}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1826485044},{"type":"v1","addr":"172.21.15.181:6829","nonce":1826485044}]},"public_addr":"172.21.15.181:6825/1826485044","cluster_addr":"172.21.15.181:6827/1826485044","heartbeat_back_addr":"172.21.15.181:6831/1826485044","heartbeat_front_addr":"172.21.15.181:6829/1826485044","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:57:52.420825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:58:23.343419+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:58:48.394844+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:12.771998+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:33.704827+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T22:59:58.445248+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T23:00:23.116899+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-12T23:00:47.489482+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.144:6801/1971060878":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/1644164096":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/289197444":"2024-04-13T22:55:58.467573+0000","172.21.15.144:0/2748859989":"2024-04-13T22:55:58.467573+0000","172.21.15.144:0/1195888470":"2024-04-13T22:56:49.008974+0000","172.21.15.144:6801/4221422342":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6800/1971060878":"2024-04-13T22:56:49.008974+0000","172.21.15.144:6800/4221422342":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6800/2478980714":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/568903382":"2024-04-13T22:56:49.008974+0000","172.21.15.144:0/3814104170":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/824159127":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/1595559936":"2024-04-13T22:55:58.467573+0000","172.21.15.144:6801/2478980714":"2024-04-13T22:55:28.720269+0000","172.21.15.144:0/1147693527":"2024-04-13T22:55:28.720269+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-12T23:01:06.137 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:05 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/131574298' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T23:01:06.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:05 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/131574298' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-12T23:01:06.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-12T23:01:06.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-12T23:01:06.508 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-12T23:01:06.508 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-12T23:01:06.508 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-12T23:01:06.509 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-12T23:01:06.509 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-12T23:01:06.510 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-12T23:01:06.948 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.950 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.951 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.952 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.952 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.953 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.953 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:06.966 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:07.177 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:06 smithi144 ceph-mon[24815]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-12T23:01:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:06 smithi181 ceph-mon[26930]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-12T23:01:08.966 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:08 smithi144 ceph-mon[24815]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:09.137 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:08 smithi181 ceph-mon[26930]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:11.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:10 smithi181 ceph-mon[26930]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:11.180 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:10 smithi144 ceph-mon[24815]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:13.180 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:12 smithi144 ceph-mon[24815]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:13.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:12 smithi181 ceph-mon[26930]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:15.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:14 smithi144 ceph-mon[24815]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:15.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:14 smithi181 ceph-mon[26930]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:16.155 INFO:teuthology.orchestra.run.smithi144.stdout:115964117019 2024-04-12T23:01:16.155 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-12T23:01:17.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:16 smithi144 ceph-mon[24815]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:17.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:16 smithi181 ceph-mon[26930]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:18.206 INFO:teuthology.orchestra.run.smithi144.stdout:38654705708 2024-04-12T23:01:18.206 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-12T23:01:19.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:18 smithi144 ceph-mon[24815]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:19.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:18 smithi181 ceph-mon[26930]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:19.494 INFO:teuthology.orchestra.run.smithi144.stdout:206158430222 2024-04-12T23:01:19.494 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-12T23:01:20.610 INFO:teuthology.orchestra.run.smithi144.stdout:240518168584 2024-04-12T23:01:20.610 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-12T23:01:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:20 smithi181 ceph-mon[26930]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:21.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:20 smithi144 ceph-mon[24815]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:21.297 INFO:teuthology.orchestra.run.smithi144.stdout:137438953496 2024-04-12T23:01:21.297 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-12T23:01:21.500 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:21.502 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:21.897 INFO:teuthology.orchestra.run.smithi144.stdout:171798691858 2024-04-12T23:01:21.897 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-12T23:01:22.926 INFO:teuthology.orchestra.run.smithi144.stdout:60129542182 2024-04-12T23:01:22.926 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-12T23:01:23.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:22 smithi181 ceph-mon[26930]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:23.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:22 smithi144 ceph-mon[24815]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:23.931 INFO:teuthology.orchestra.run.smithi144.stdout:81604378658 2024-04-12T23:01:23.931 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-12T23:01:24.126 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:24.126 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:24.773 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:25.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:24 smithi181 ceph-mon[26930]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:25.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:24 smithi144 ceph-mon[24815]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:26.878 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:26.878 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:26.879 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:27.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:26 smithi144 ceph-mon[24815]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:27.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:26 smithi181 ceph-mon[26930]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:28.225 INFO:teuthology.orchestra.run.smithi144.stdout:38654705710 2024-04-12T23:01:29.185 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:28 smithi144 ceph-mon[24815]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:29.186 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:28 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2194825248' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T23:01:29.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:28 smithi181 ceph-mon[26930]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:29.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:28 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2194825248' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-12T23:01:29.944 INFO:teuthology.orchestra.run.smithi144.stdout:115964117022 2024-04-12T23:01:30.615 INFO:teuthology.orchestra.run.smithi144.stdout:206158430224 2024-04-12T23:01:31.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:30 smithi144 ceph-mon[24815]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:31.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2349402423' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T23:01:31.157 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1857555989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T23:01:31.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:30 smithi181 ceph-mon[26930]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:31.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2349402423' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-12T23:01:31.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1857555989' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-12T23:01:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:32 smithi181 ceph-mon[26930]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:32 smithi144 ceph-mon[24815]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:33.410 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705710 for osd.0 2024-04-12T23:01:33.411 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:33.780 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117019 got 115964117022 for osd.3 2024-04-12T23:01:33.781 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:34.493 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430224 for osd.6 2024-04-12T23:01:34.493 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:35.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:34 smithi144 ceph-mon[24815]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:35.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:34 smithi181 ceph-mon[26930]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:35.315 INFO:teuthology.orchestra.run.smithi144.stdout:60129542184 2024-04-12T23:01:35.499 INFO:teuthology.orchestra.run.smithi144.stdout:137438953499 2024-04-12T23:01:36.072 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:35 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/4140944550' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T23:01:36.072 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:35 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/4074056186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T23:01:36.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:35 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/4140944550' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-12T23:01:36.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:35 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/4074056186' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-12T23:01:36.540 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542184 for osd.1 2024-04-12T23:01:36.540 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:37.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:36 smithi144 ceph-mon[24815]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:37.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:36 smithi181 ceph-mon[26930]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:37.315 INFO:teuthology.orchestra.run.smithi144.stdout:81604378660 2024-04-12T23:01:37.509 INFO:teuthology.orchestra.run.smithi144.stdout:240518168588 2024-04-12T23:01:38.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:37 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3661530280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T23:01:38.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:37 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3777180007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T23:01:38.246 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:37 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3661530280' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-12T23:01:38.246 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:37 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3777180007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-12T23:01:38.271 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953496 got 137438953499 for osd.4 2024-04-12T23:01:38.272 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:38.660 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168584 got 240518168588 for osd.7 2024-04-12T23:01:38.660 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:39.145 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378660 for osd.2 2024-04-12T23:01:39.145 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:39.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:38 smithi181 ceph-mon[26930]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:39.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:38 smithi144 ceph-mon[24815]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:39.701 INFO:teuthology.orchestra.run.smithi144.stdout:171798691862 2024-04-12T23:01:40.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:39 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3345546362' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T23:01:40.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:39 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3345546362' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-12T23:01:40.206 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691858 got 171798691862 for osd.5 2024-04-12T23:01:40.206 DEBUG:teuthology.parallel:result is None 2024-04-12T23:01:40.206 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-12T23:01:40.206 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-12T23:01:40.421 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:41.190 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:40 smithi144 ceph-mon[24815]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:41.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:40 smithi181 ceph-mon[26930]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:42.112 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:01:42.114 INFO:teuthology.orchestra.run.smithi144.stderr:dumped all 2024-04-12T23:01:42.710 INFO:teuthology.orchestra.run.smithi144.stdout:{"pg_ready":true,"pg_map":{"version":194,"stamp":"2024-04-12T23:01:41.756567+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49012,"kb_used_data":6388,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912204,"statfs":{"total":767960285184,"available":767910096896,"internally_reserved":0,"allocated":6541312,"data_stored":3572655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002028"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-12T23:00:52.761633+0000","last_change":"2024-04-12T23:00:52.761633+0000","last_active":"2024-04-12T23:00:52.761633+0000","last_peered":"2024-04-12T23:00:52.761633+0000","last_clean":"2024-04-12T23:00:52.761633+0000","last_became_active":"2024-04-12T23:00:52.761172+0000","last_became_peered":"2024-04-12T23:00:52.761172+0000","last_unstale":"2024-04-12T23:00:52.761633+0000","last_undegraded":"2024-04-12T23:00:52.761633+0000","last_fullsized":"2024-04-12T23:00:52.761633+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T22:58:52.058027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T22:58:52.058027+0000","last_clean_scrub_stamp":"2024-04-12T22:58:52.058027+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T01:31:23.734021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52700000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56699999999999995}]},{"osd":1,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.436},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.59999999999999998}]},{"osd":2,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.50600000000000001}]},{"osd":3,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64700000000000002}]},{"osd":4,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.626},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.72899999999999998}]},{"osd":5,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542186,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.34899999999999998,"15min":0.309},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.37,"15min":0.32500000000000001},"min":{"1min":0.25900000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.29,"5min":2.29,"15min":2.29},"last":0.35799999999999998}]},{"osd":2,"last update":"Fri Apr 12 23:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.376,"15min":0.36399999999999999},"min":{"1min":0.29699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.377,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.39800000000000002}]},{"osd":3,"last update":"Fri Apr 12 23:01:18 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.48099999999999998,"15min":0.46500000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.5089999999999999,"5min":2.5089999999999999,"15min":2.5089999999999999},"last":0.442},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.51600000000000001,"15min":0.504},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.7989999999999999,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.60799999999999998,"15min":0.59399999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.3690000000000002,"5min":2.3690000000000002,"15min":2.3690000000000002},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.628,"15min":0.61599999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":0.65300000000000002}]},{"osd":5,"last update":"Fri Apr 12 23:01:01 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66600000000000004}]},{"osd":6,"last update":"Fri Apr 12 23:01:29 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.1240000000000001,"5min":2.1240000000000001,"15min":2.1240000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.76000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 23:01:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.34599999999999997,"15min":0.32800000000000001},"min":{"1min":0.26700000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.371,"15min":0.35299999999999998},"min":{"1min":0.25800000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.59299999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.376}]},{"osd":2,"last update":"Fri Apr 12 23:00:56 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.314},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.378,"15min":0.374},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.25}]},{"osd":3,"last update":"Fri Apr 12 23:01:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42299999999999999,"15min":0.42499999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.33300000000000002}]},{"osd":4,"last update":"Fri Apr 12 23:01:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.57999999999999996,"15min":0.57799999999999996},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42099999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.58299999999999996}]},{"osd":5,"last update":"Fri Apr 12 23:01:00 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.64300000000000002}]},{"osd":6,"last update":"Fri Apr 12 23:01:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.621},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.54700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68899999999999995}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.28899999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.379,"15min":0.374},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34799999999999998}]},{"osd":1,"last update":"Fri Apr 12 23:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.39700000000000002,"15min":0.39300000000000002},"min":{"1min":0.316,"5min":0.249,"15min":0.249},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.38100000000000001},"min":{"1min":0.32900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.33800000000000002}]},{"osd":3,"last update":"Fri Apr 12 23:01:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.372}]},{"osd":4,"last update":"Fri Apr 12 23:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58299999999999996,"15min":0.58499999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.71999999999999997}]},{"osd":5,"last update":"Fri Apr 12 23:01:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.63100000000000001}]},{"osd":6,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.747},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.56999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68000000000000005}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:19 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.39800000000000002,"15min":0.40600000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.434,"15min":0.436},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.34999999999999998}]},{"osd":1,"last update":"Fri Apr 12 23:01:19 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40000000000000002,"15min":0.40400000000000003},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.45200000000000001,"15min":0.46100000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.57899999999999996,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Fri Apr 12 23:01:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.47599999999999998,"15min":0.48699999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57699999999999996,"5min":0.62,"15min":0.62},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.501,"15min":0.51100000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.64300000000000002,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.42099999999999999}]},{"osd":4,"last update":"Fri Apr 12 23:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.44,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68799999999999994,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.66700000000000004}]},{"osd":5,"last update":"Fri Apr 12 23:01:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.56799999999999995}]},{"osd":6,"last update":"Fri Apr 12 23:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.751},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.68300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.39200000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.496},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.51400000000000001,"15min":0.50900000000000001},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.44600000000000001}]},{"osd":1,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.38600000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.52500000000000002,"15min":0.52300000000000002},"min":{"1min":0.373,"5min":0.307,"15min":0.307},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.46800000000000003}]},{"osd":2,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.36899999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.42399999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.63200000000000001}]},{"osd":3,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.53600000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.67200000000000004}]},{"osd":5,"last update":"Fri Apr 12 23:01:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.625},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.61699999999999999}]},{"osd":6,"last update":"Fri Apr 12 23:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.60399999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65100000000000002}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.44400000000000001}]},{"osd":1,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.242,"5min":1.242,"15min":1.242},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":16.818000000000001,"5min":16.818000000000001,"15min":16.818000000000001},"last":0.51400000000000001}]},{"osd":3,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.625}]},{"osd":4,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.502}]},{"osd":6,"last update":"Fri Apr 12 23:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.69699999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T23:01:42.712 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-12T23:01:42.925 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:43.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:42 smithi144 ceph-mon[24815]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:43.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:42 smithi144 ceph-mon[24815]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:01:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:42 smithi181 ceph-mon[26930]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:42 smithi181 ceph-mon[26930]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:01:44.548 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:01:44.550 INFO:teuthology.orchestra.run.smithi144.stderr:dumped all 2024-04-12T23:01:45.193 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:44 smithi144 ceph-mon[24815]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:44 smithi181 ceph-mon[26930]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:45.218 INFO:teuthology.orchestra.run.smithi144.stdout:{"pg_ready":true,"pg_map":{"version":195,"stamp":"2024-04-12T23:01:43.756936+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49012,"kb_used_data":6388,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912204,"statfs":{"total":767960285184,"available":767910096896,"internally_reserved":0,"allocated":6541312,"data_stored":3572655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002072"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-12T23:00:52.761633+0000","last_change":"2024-04-12T23:00:52.761633+0000","last_active":"2024-04-12T23:00:52.761633+0000","last_peered":"2024-04-12T23:00:52.761633+0000","last_clean":"2024-04-12T23:00:52.761633+0000","last_became_active":"2024-04-12T23:00:52.761172+0000","last_became_peered":"2024-04-12T23:00:52.761172+0000","last_unstale":"2024-04-12T23:00:52.761633+0000","last_undegraded":"2024-04-12T23:00:52.761633+0000","last_fullsized":"2024-04-12T23:00:52.761633+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-12T22:58:52.058027+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-12T22:58:52.058027+0000","last_clean_scrub_stamp":"2024-04-12T22:58:52.058027+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T01:31:23.734021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168589,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46300000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52700000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.56699999999999995}]},{"osd":1,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.436},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.59999999999999998}]},{"osd":2,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.50600000000000001}]},{"osd":3,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.64700000000000002}]},{"osd":4,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.626},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.72899999999999998}]},{"osd":5,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.69799999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":1,"up_from":14,"seq":60129542186,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.34899999999999998,"15min":0.309},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":1.8600000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.37,"15min":0.32500000000000001},"min":{"1min":0.25900000000000001,"5min":0.215,"15min":0.215},"max":{"1min":2.29,"5min":2.29,"15min":2.29},"last":0.35799999999999998}]},{"osd":2,"last update":"Fri Apr 12 23:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.376,"15min":0.36399999999999999},"min":{"1min":0.29699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.377,"15min":0.36899999999999999},"min":{"1min":0.26000000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.39800000000000002}]},{"osd":3,"last update":"Fri Apr 12 23:01:18 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.48099999999999998,"15min":0.46500000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.5089999999999999,"5min":2.5089999999999999,"15min":2.5089999999999999},"last":0.442},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.51600000000000001,"15min":0.504},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.7989999999999999,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":0.42199999999999999}]},{"osd":4,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.60799999999999998,"15min":0.59399999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.3690000000000002,"5min":2.3690000000000002,"15min":2.3690000000000002},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.628,"15min":0.61599999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":2.1920000000000002,"5min":2.1920000000000002,"15min":2.1920000000000002},"last":0.65300000000000002}]},{"osd":5,"last update":"Fri Apr 12 23:01:01 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.66600000000000004}]},{"osd":6,"last update":"Fri Apr 12 23:01:29 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.1240000000000001,"5min":2.1240000000000001,"15min":2.1240000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.76000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 12 23:01:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.34599999999999997,"15min":0.32800000000000001},"min":{"1min":0.26700000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.371,"15min":0.35299999999999998},"min":{"1min":0.25800000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.59299999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.376}]},{"osd":2,"last update":"Fri Apr 12 23:00:56 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.40000000000000002,"15min":0.39400000000000002},"min":{"1min":0.28799999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.314},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.378,"15min":0.374},"min":{"1min":0.27200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.25}]},{"osd":3,"last update":"Fri Apr 12 23:01:15 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.316,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42299999999999999,"15min":0.42499999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.33300000000000002}]},{"osd":4,"last update":"Fri Apr 12 23:01:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.57999999999999996,"15min":0.57799999999999996},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42099999999999999,"5min":0.376,"15min":0.376},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.58299999999999996}]},{"osd":5,"last update":"Fri Apr 12 23:01:00 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.64300000000000002}]},{"osd":6,"last update":"Fri Apr 12 23:01:26 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.621},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.54700000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68899999999999995}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.378,"15min":0.375},"min":{"1min":0.28899999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.379,"15min":0.374},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34799999999999998}]},{"osd":1,"last update":"Fri Apr 12 23:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.39700000000000002,"15min":0.39300000000000002},"min":{"1min":0.316,"5min":0.249,"15min":0.249},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38800000000000001,"15min":0.38100000000000001},"min":{"1min":0.32900000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.33800000000000002}]},{"osd":3,"last update":"Fri Apr 12 23:01:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.372}]},{"osd":4,"last update":"Fri Apr 12 23:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58299999999999996,"15min":0.58499999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56999999999999995,"15min":0.56799999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.71999999999999997}]},{"osd":5,"last update":"Fri Apr 12 23:01:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.63100000000000001}]},{"osd":6,"last update":"Fri Apr 12 23:01:25 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.747},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.56999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68000000000000005}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:19 2024","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.39800000000000002,"15min":0.40600000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.434,"15min":0.436},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.34999999999999998}]},{"osd":1,"last update":"Fri Apr 12 23:01:19 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.40000000000000002,"15min":0.40400000000000003},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.45200000000000001,"15min":0.46100000000000002},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.57899999999999996,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.38800000000000001}]},{"osd":2,"last update":"Fri Apr 12 23:01:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.47599999999999998,"15min":0.48699999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57699999999999996,"5min":0.62,"15min":0.62},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.501,"15min":0.51100000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.64300000000000002,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.42099999999999999}]},{"osd":4,"last update":"Fri Apr 12 23:01:40 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.44,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68799999999999994,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.66700000000000004}]},{"osd":5,"last update":"Fri Apr 12 23:01:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.56799999999999995}]},{"osd":6,"last update":"Fri Apr 12 23:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.751},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.68300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5728,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739424,"statfs":{"total":95995035648,"available":95989170176,"internally_reserved":0,"allocated":614400,"data_stored":248002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.39200000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.496},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.51400000000000001,"15min":0.50900000000000001},"min":{"1min":0.40200000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.44600000000000001}]},{"osd":1,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.38600000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.52500000000000002,"15min":0.52300000000000002},"min":{"1min":0.373,"5min":0.307,"15min":0.307},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.46800000000000003}]},{"osd":2,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.36899999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.42399999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.63200000000000001}]},{"osd":3,"last update":"Fri Apr 12 23:01:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.53600000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.67200000000000004}]},{"osd":5,"last update":"Fri Apr 12 23:01:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.625},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.61699999999999999}]},{"osd":6,"last update":"Fri Apr 12 23:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.60399999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65100000000000002}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.44400000000000001}]},{"osd":1,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.49399999999999999}]},{"osd":2,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.242,"5min":1.242,"15min":1.242},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":16.818000000000001,"5min":16.818000000000001,"15min":16.818000000000001},"last":0.51400000000000001}]},{"osd":3,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.625}]},{"osd":4,"last update":"Fri Apr 12 23:01:04 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.502}]},{"osd":6,"last update":"Fri Apr 12 23:01:27 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.69699999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-12T23:01:45.219 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-12T23:01:45.219 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-12T23:01:45.219 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-12T23:01:45.220 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-12T23:01:45.436 INFO:teuthology.orchestra.run.smithi144.stderr:Inferring config /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/mon.smithi144/config 2024-04-12T23:01:46.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:45 smithi181 ceph-mon[26930]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:01:46.224 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:45 smithi144 ceph-mon[24815]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:01:47.039 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:46 smithi144 ceph-mon[24815]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:46 smithi181 ceph-mon[26930]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:47.204 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:01:47.204 INFO:teuthology.orchestra.run.smithi144.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-12T23:01:47.805 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-12T23:01:47.806 INFO:tasks.cephadm:Setup complete, yielding 2024-04-12T23:01:47.806 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T23:01:47.816 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:01:47.816 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch status' 2024-04-12T23:01:48.126 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:47 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/996781932' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T23:01:48.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:47 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/996781932' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-12T23:01:49.145 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:48 smithi144 ceph-mon[24815]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:49.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:48 smithi181 ceph-mon[26930]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:49.577 INFO:teuthology.orchestra.run.smithi144.stdout:Backend: cephadm 2024-04-12T23:01:49.578 INFO:teuthology.orchestra.run.smithi144.stdout:Available: Yes 2024-04-12T23:01:49.578 INFO:teuthology.orchestra.run.smithi144.stdout:Paused: No 2024-04-12T23:01:50.195 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:01:50.195 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:01:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:01:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:01:50.219 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch ps' 2024-04-12T23:01:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:50 smithi144 ceph-mon[24815]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:50 smithi144 ceph-mon[24815]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:01:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:01:51.140 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:01:51.152 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:50 smithi181 ceph-mon[26930]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:50 smithi181 ceph-mon[26930]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:01:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:01:51.153 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (4m) 2m ago 5m 13.3M - ba2b418f427c 5fe9fb07ff47 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (4m) 64s ago 4m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (3m) 2m ago 5m 45.8M - 8.3.5 dad864ee21e9 79755181f90b 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:01:52.064 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (4m) 64s ago 4m 410M - 17.2.0 e1d6a67b021e bd511fd5a495 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (6m) 2m ago 6m 43.0M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (4m) 64s ago 4m 39.7M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (5m) 2m ago 5m 18.2M - 1dbe0e931976 7410a549b2f7 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (4m) 64s ago 4m 19.3M - 1dbe0e931976 0d304d76f102 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (4m) 2m ago 4m 33.1M 3216M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (3m) 2m ago 3m 35.5M 3216M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (3m) 2m ago 3m 31.5M 3216M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (2m) 2m ago 2m 10.2M 3216M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (2m) 64s ago 2m 34.0M 3984M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (118s) 64s ago 117s 33.5M 3984M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (93s) 64s ago 93s 33.0M 3984M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (69s) 64s ago 68s 10.9M 3984M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:01:52.065 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (4m) 2m ago 4m 51.5M - 514e6a882f6e 3c5300d666ec 2024-04-12T23:01:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:52 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:01:52.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:52 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:01:52.691 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch ls' 2024-04-12T23:01:53.249 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:53 smithi144 ceph-mon[24815]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:53.249 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:53 smithi144 ceph-mon[24815]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:53.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:53 smithi181 ceph-mon[26930]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:53.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:53 smithi181 ceph-mon[26930]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:crash 2/2 2m ago 6m * 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:mgr 2/2 2m ago 6m count:2 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:mon 2/2 2m ago 5m smithi144:172.21.15.144=smithi144;smithi181:172.21.15.181=smithi181;count:2 2024-04-12T23:01:54.542 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-04-12T23:01:54.543 INFO:teuthology.orchestra.run.smithi144.stdout:osd 8 2m ago - 2024-04-12T23:01:54.543 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-04-12T23:01:55.222 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch host ls' 2024-04-12T23:01:55.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:55 smithi181 ceph-mon[26930]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:55.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:55 smithi144 ceph-mon[24815]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:56.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:56 smithi181 ceph-mon[26930]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:56.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:56 smithi144 ceph-mon[24815]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:57.059 INFO:teuthology.orchestra.run.smithi144.stdout:HOST ADDR LABELS STATUS 2024-04-12T23:01:57.059 INFO:teuthology.orchestra.run.smithi144.stdout:smithi144 172.21.15.144 2024-04-12T23:01:57.059 INFO:teuthology.orchestra.run.smithi144.stdout:smithi181 172.21.15.181 2024-04-12T23:01:57.059 INFO:teuthology.orchestra.run.smithi144.stdout:2 hosts in cluster 2024-04-12T23:01:57.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:57 smithi181 ceph-mon[26930]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:57.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:57 smithi144 ceph-mon[24815]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:57.685 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph orch device ls' 2024-04-12T23:01:58.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:58 smithi144 ceph-mon[24815]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:58.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:58 smithi181 ceph-mon[26930]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:01:59.404 INFO:teuthology.orchestra.run.smithi144.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-04-12T23:01:59.405 INFO:teuthology.orchestra.run.smithi144.stdout:smithi144 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62050085400BGN 400G LVM detected, locked 2024-04-12T23:01:59.405 INFO:teuthology.orchestra.run.smithi144.stdout:smithi181 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6246000E400BGN 400G LVM detected, locked 2024-04-12T23:01:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:01:59 smithi181 ceph-mon[26930]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:59.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:01:59 smithi144 ceph-mon[24815]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:01:59.981 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-12T23:01:59.991 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:01:59.991 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-12T23:02:00.020 INFO:teuthology.orchestra.run.smithi144.stderr:+ systemctl stop nfs-server 2024-04-12T23:02:00.027 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi181.front.sepia.ceph.com 2024-04-12T23:02:00.027 DEBUG:teuthology.orchestra.run.smithi181:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-04-12T23:02:00.059 INFO:teuthology.orchestra.run.smithi181.stderr:+ systemctl stop nfs-server 2024-04-12T23:02:00.068 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T23:02:00.077 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:02:00.077 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-04-12T23:02:01.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:01 smithi144 ceph-mon[24815]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:01.337 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:01 smithi144 ceph-mon[24815]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:01.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:01 smithi181 ceph-mon[26930]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:01.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:01 smithi181 ceph-mon[26930]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:02 smithi181 ceph-mon[26930]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:02 smithi181 ceph-mon[26930]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-12T23:02:03.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:02 smithi144 ceph-mon[24815]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:03.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:02 smithi144 ceph-mon[24815]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:03.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[24811]: 2024-04-12T23:02:03.351+0000 7fb9477aa700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: fsmap foofs:0 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:03.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T23:02:03.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-04-12T23:02:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: osdmap e61: 8 total, 8 up, 8 in 2024-04-12T23:02:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: osdmap e62: 8 total, 8 up, 8 in 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: osdmap e63: 8 total, 8 up, 8 in 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: fsmap foofs:0 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:03.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:03.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:02:03.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T23:02:03.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:04.002 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-04-12T23:02:04.011 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-04-12T23:02:04.012 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T23:02:05.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:04 smithi144 ceph-mon[24815]: Saving service mds.foofs spec with placement count:2 2024-04-12T23:02:05.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:04 smithi144 ceph-mon[24815]: Deploying daemon mds.foofs.smithi144.alilcy on smithi144 2024-04-12T23:02:05.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:04 smithi144 ceph-mon[24815]: pgmap v208: 65 pgs: 64 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:04 smithi144 ceph-mon[24815]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T23:02:05.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:04 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:04 smithi181 ceph-mon[26930]: Saving service mds.foofs spec with placement count:2 2024-04-12T23:02:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:04 smithi181 ceph-mon[26930]: Deploying daemon mds.foofs.smithi144.alilcy on smithi144 2024-04-12T23:02:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:04 smithi181 ceph-mon[26930]: pgmap v208: 65 pgs: 64 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:04 smithi181 ceph-mon[26930]: osdmap e64: 8 total, 8 up, 8 in 2024-04-12T23:02:05.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:04 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:06.661 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:06 smithi144 ceph-mon[24815]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T23:02:06.662 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:06 smithi144 ceph-mon[24815]: pgmap v211: 65 pgs: 6 creating+peering, 16 unknown, 43 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:06.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:06 smithi181 ceph-mon[26930]: osdmap e65: 8 total, 8 up, 8 in 2024-04-12T23:02:06.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:06 smithi181 ceph-mon[26930]: pgmap v211: 65 pgs: 6 creating+peering, 16 unknown, 43 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:06.880 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:02:06.880 INFO:teuthology.orchestra.run.smithi144.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T22:55:48.243697Z", "last_refresh": "2024-04-12T22:59:15.221042Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T22:55:42.567016Z", "last_refresh": "2024-04-12T22:59:15.221178Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T22:55:45.775050Z", "last_refresh": "2024-04-12T22:59:15.221289Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T23:02:03.372280Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T23:02:03.365164Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T22:55:40.927783Z", "last_refresh": "2024-04-12T22:59:15.220912Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi144:172.21.15.144=smithi144", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T22:56:30.064757Z", "last_refresh": "2024-04-12T22:59:15.220687Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T22:55:46.969530Z", "last_refresh": "2024-04-12T22:59:15.221392Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T22:59:15.221637Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T22:55:44.565108Z", "last_refresh": "2024-04-12T22:59:15.221501Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T23:02:07.618 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:07 smithi144 ceph-mon[24815]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:07.642 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-12T23:02:07.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:07 smithi181 ceph-mon[26930]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:08.642 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T23:02:08.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: pgmap v212: 65 pgs: 6 creating+peering, 59 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:08.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:08.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:02:08.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T23:02:08.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: Deploying daemon mds.foofs.smithi181.gciyns on smithi181 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: mds.? [v2:172.21.15.144:6834/778305732,v1:172.21.15.144:6835/778305732] up:boot 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: daemon mds.foofs.smithi144.alilcy assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: Cluster is now healthy 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: fsmap foofs:0 1 up:standby 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:creating} 2024-04-12T23:02:08.954 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:08 smithi144 ceph-mon[24815]: daemon mds.foofs.smithi144.alilcy is now active in filesystem foofs as rank 0 2024-04-12T23:02:09.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: pgmap v212: 65 pgs: 6 creating+peering, 59 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:09.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: Deploying daemon mds.foofs.smithi181.gciyns on smithi181 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: mds.? [v2:172.21.15.144:6834/778305732,v1:172.21.15.144:6835/778305732] up:boot 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: daemon mds.foofs.smithi144.alilcy assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: Cluster is now healthy 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: fsmap foofs:0 1 up:standby 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:creating} 2024-04-12T23:02:09.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:08 smithi181 ceph-mon[26930]: daemon mds.foofs.smithi144.alilcy is now active in filesystem foofs as rank 0 2024-04-12T23:02:10.513 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:10 smithi144 ceph-mon[24815]: mds.? [v2:172.21.15.144:6834/778305732,v1:172.21.15.144:6835/778305732] up:active 2024-04-12T23:02:10.513 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:10 smithi144 ceph-mon[24815]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 2024-04-12T23:02:10.513 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:10 smithi144 ceph-mon[24815]: pgmap v213: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-04-12T23:02:10.534 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:02:10.534 INFO:teuthology.orchestra.run.smithi144.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T22:55:48.243697Z", "last_refresh": "2024-04-12T22:59:15.221042Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T22:55:42.567016Z", "last_refresh": "2024-04-12T22:59:15.221178Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T22:55:45.775050Z", "last_refresh": "2024-04-12T22:59:15.221289Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T23:02:03.372280Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T23:02:03.365164Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T22:55:40.927783Z", "last_refresh": "2024-04-12T22:59:15.220912Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi144:172.21.15.144=smithi144", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T22:56:30.064757Z", "last_refresh": "2024-04-12T22:59:15.220687Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T22:55:46.969530Z", "last_refresh": "2024-04-12T22:59:15.221392Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T22:59:15.221637Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T22:55:44.565108Z", "last_refresh": "2024-04-12T22:59:15.221501Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T23:02:10.567 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:10 smithi181 ceph-mon[26930]: mds.? [v2:172.21.15.144:6834/778305732,v1:172.21.15.144:6835/778305732] up:active 2024-04-12T23:02:10.567 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:10 smithi181 ceph-mon[26930]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 2024-04-12T23:02:10.567 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:10 smithi181 ceph-mon[26930]: pgmap v213: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 2 op/s 2024-04-12T23:02:11.114 INFO:tasks.cephadm:mds.foofs has 0/2 2024-04-12T23:02:11.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:boot 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:11.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:11 smithi181 ceph-mon[26930]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:11.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:boot 2024-04-12T23:02:11.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:02:11.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:02:11.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:02:11.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:11.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:11.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:11.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:11.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:11 smithi144 ceph-mon[24815]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:12.115 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T23:02:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:12 smithi181 ceph-mon[26930]: pgmap v214: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-04-12T23:02:12.708 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:12 smithi144 ceph-mon[24815]: pgmap v214: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-04-12T23:02:13.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:13 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:13.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:13 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:14.673 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:02:14.673 INFO:teuthology.orchestra.run.smithi144.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T22:55:48.243697Z", "last_refresh": "2024-04-12T22:59:15.221042Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T22:55:42.567016Z", "last_refresh": "2024-04-12T22:59:15.221178Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T22:55:45.775050Z", "last_refresh": "2024-04-12T22:59:15.221289Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T23:02:03.372280Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T23:02:03.365164Z", "last_refresh": "2024-04-12T23:02:12.450924Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T22:55:40.927783Z", "last_refresh": "2024-04-12T22:59:15.220912Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi144:172.21.15.144=smithi144", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T22:56:30.064757Z", "last_refresh": "2024-04-12T22:59:15.220687Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T22:55:46.969530Z", "last_refresh": "2024-04-12T22:59:15.221392Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T22:59:15.221637Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T22:55:44.565108Z", "last_refresh": "2024-04-12T22:59:15.221501Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T23:02:15.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:14 smithi144 ceph-mon[24815]: pgmap v215: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T23:02:15.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:14 smithi181 ceph-mon[26930]: pgmap v215: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-04-12T23:02:15.420 INFO:tasks.cephadm:mds.foofs has 1/2 2024-04-12T23:02:16.103 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:15 smithi144 ceph-mon[24815]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:16.103 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:15 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:16.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:15 smithi181 ceph-mon[26930]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:16.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:15 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:16.421 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:16.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:16 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:17.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: pgmap v216: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-04-12T23:02:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:17.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:16 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:18.421 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:02:18.421 INFO:teuthology.orchestra.run.smithi144.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T22:55:48.243697Z", "last_refresh": "2024-04-12T23:02:16.679453Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T22:55:42.567016Z", "last_refresh": "2024-04-12T23:02:12.450274Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-04-12T22:55:45.775050Z", "last_refresh": "2024-04-12T23:02:16.679589Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-04-12T23:02:03.372280Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T23:02:03.365164Z", "last_refresh": "2024-04-12T23:02:12.450924Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T22:55:40.927783Z", "last_refresh": "2024-04-12T23:02:12.450428Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi144:172.21.15.144=smithi144", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T22:56:30.064757Z", "last_refresh": "2024-04-12T23:02:12.450499Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T22:55:46.969530Z", "last_refresh": "2024-04-12T23:02:12.450562Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-04-12T23:02:12.450657Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T22:55:44.565108Z", "last_refresh": "2024-04-12T23:02:16.679715Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T23:02:19.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:18 smithi144 ceph-mon[24815]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T23:02:19.086 INFO:tasks.cephadm:mds.foofs has 2/2 2024-04-12T23:02:19.087 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T23:02:19.096 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:02:19.096 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-04-12T23:02:19.136 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:18 smithi181 ceph-mon[26930]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T23:02:20.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:19 smithi144 ceph-mon[24815]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:20.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:19 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:20.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:19 smithi181 ceph-mon[26930]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:02:20.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:19 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:21.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:20 smithi144 ceph-mon[24815]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T23:02:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:20 smithi181 ceph-mon[26930]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-12T23:02:21.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:21 smithi181 ceph-mon[26930]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:21.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:21 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-12T23:02:22.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:21 smithi144 ceph-mon[24815]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:22.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:21 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-04-12T23:02:22.831 INFO:teuthology.orchestra.run.smithi144.stdout:NFS Cluster Created Successfully 2024-04-12T23:02:23.053 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:22 smithi144 ceph-mon[24815]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 2 op/s 2024-04-12T23:02:23.053 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:22 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-12T23:02:23.053 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:22 smithi144 ceph-mon[24815]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T23:02:23.053 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:22 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-12T23:02:23.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:22 smithi181 ceph-mon[26930]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 2 op/s 2024-04-12T23:02:23.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:22 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-04-12T23:02:23.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:22 smithi181 ceph-mon[26930]: osdmap e66: 8 total, 8 up, 8 in 2024-04-12T23:02:23.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:22 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-04-12T23:02:23.585 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-04-12T23:02:24.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-12T23:02:24.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T23:02:24.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: Saving service nfs.foo spec with placement count:2 2024-04-12T23:02:24.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:24.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:02:24.090 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: Rados config object exists: conf-nfs.foo 2024-04-12T23:02:24.091 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt-rgw 2024-04-12T23:02:24.091 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:02:24.091 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T23:02:24.091 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:24.091 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:23 smithi144 ceph-mon[24815]: Deploying daemon nfs.foo.0.0.smithi144.mjwmkt on smithi144 2024-04-12T23:02:24.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-04-12T23:02:24.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: osdmap e67: 8 total, 8 up, 8 in 2024-04-12T23:02:24.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: Saving service nfs.foo spec with placement count:2 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:02:24.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:02:24.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: Rados config object exists: conf-nfs.foo 2024-04-12T23:02:24.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt-rgw 2024-04-12T23:02:24.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:02:24.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T23:02:24.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:24.200 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:23 smithi181 ceph-mon[26930]: Deploying daemon nfs.foo.0.0.smithi144.mjwmkt on smithi144 2024-04-12T23:02:24.968 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:24 smithi144 ceph-mon[24815]: pgmap v222: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:24.968 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:24 smithi144 ceph-mon[24815]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T23:02:24.968 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:24 smithi144 ceph-mon[24815]: mgrmap e20: smithi144.rlueky(active, since 5m), standbys: smithi181.whsrmz 2024-04-12T23:02:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:24 smithi181 ceph-mon[26930]: pgmap v222: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-04-12T23:02:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:24 smithi181 ceph-mon[26930]: osdmap e68: 8 total, 8 up, 8 in 2024-04-12T23:02:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:24 smithi181 ceph-mon[26930]: mgrmap e20: smithi144.rlueky(active, since 5m), standbys: smithi181.whsrmz 2024-04-12T23:02:26.666 INFO:teuthology.orchestra.run.smithi144.stderr:Invalid command: Unexpected argument '--clusterid' 2024-04-12T23:02:26.666 INFO:teuthology.orchestra.run.smithi144.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-04-12T23:02:26.666 INFO:teuthology.orchestra.run.smithi144.stderr:Error EINVAL: invalid command 2024-04-12T23:02:27.013 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:02:27.013 INFO:teuthology.orchestra.run.smithi144.stdout: "bind": "/fake", 2024-04-12T23:02:27.013 INFO:teuthology.orchestra.run.smithi144.stdout: "fs": "foofs", 2024-04-12T23:02:27.014 INFO:teuthology.orchestra.run.smithi144.stdout: "path": "/", 2024-04-12T23:02:27.014 INFO:teuthology.orchestra.run.smithi144.stdout: "cluster": "foo", 2024-04-12T23:02:27.014 INFO:teuthology.orchestra.run.smithi144.stdout: "mode": "RW" 2024-04-12T23:02:27.014 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:02:27.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:26 smithi181 ceph-mon[26930]: pgmap v224: 97 pgs: 13 unknown, 84 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T23:02:27.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:26 smithi144 ceph-mon[24815]: pgmap v224: 97 pgs: 13 unknown, 84 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T23:02:27.769 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-04-12T23:02:28.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:28.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb 2024-04-12T23:02:28.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='client.14532 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-12T23:02:28.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:27 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-12T23:02:28.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:28.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb 2024-04-12T23:02:28.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:02:28.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-04-12T23:02:28.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T23:02:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:02:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:02:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='client.14532 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-04-12T23:02:28.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:27 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-04-12T23:02:29.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:29 smithi144 ceph-mon[24815]: pgmap v225: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T23:02:29.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:29 smithi181 ceph-mon[26930]: pgmap v225: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-12T23:02:30.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:02:30.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:02:30.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:02:30.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T23:02:30.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:30 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:30.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:02:30.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:02:30.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:02:30.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-12T23:02:30.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:30 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:31.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:31 smithi144 ceph-mon[24815]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:31.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:31 smithi144 ceph-mon[24815]: pgmap v226: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 257 B/s wr, 0 op/s 2024-04-12T23:02:31.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:31 smithi144 ceph-mon[24815]: Rados config object exists: conf-nfs.foo 2024-04-12T23:02:31.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:31 smithi144 ceph-mon[24815]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb-rgw 2024-04-12T23:02:31.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:31 smithi144 ceph-mon[24815]: Deploying daemon nfs.foo.1.0.smithi181.dgxqfb on smithi181 2024-04-12T23:02:31.304 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:31 smithi181 ceph-mon[26930]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:31.304 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:31 smithi181 ceph-mon[26930]: pgmap v226: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 128 B/s rd, 257 B/s wr, 0 op/s 2024-04-12T23:02:31.304 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:31 smithi181 ceph-mon[26930]: Rados config object exists: conf-nfs.foo 2024-04-12T23:02:31.304 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:31 smithi181 ceph-mon[26930]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb-rgw 2024-04-12T23:02:31.304 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:31 smithi181 ceph-mon[26930]: Deploying daemon nfs.foo.1.0.smithi181.dgxqfb on smithi181 2024-04-12T23:02:32.277 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:32 smithi144 ceph-mon[24815]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:32.285 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:32 smithi181 ceph-mon[26930]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:33 smithi181 ceph-mon[26930]: pgmap v227: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-12T23:02:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:33.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:33 smithi144 ceph-mon[24815]: pgmap v227: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-12T23:02:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:33.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:34.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:34 smithi144 ceph-mon[24815]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:34 smithi181 ceph-mon[26930]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:35.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:35 smithi181 ceph-mon[26930]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:35.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:35 smithi181 ceph-mon[26930]: pgmap v228: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-12T23:02:35.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:35 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:35.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:35 smithi144 ceph-mon[24815]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:35.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:35 smithi144 ceph-mon[24815]: pgmap v228: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-12T23:02:35.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:35 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:36.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:36 smithi144 ceph-mon[24815]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:36.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:36 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:36.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:36 smithi181 ceph-mon[26930]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:36.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:36 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:37.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:37 smithi181 ceph-mon[26930]: pgmap v229: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-04-12T23:02:37.524 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:37 smithi144 ceph-mon[24815]: pgmap v229: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-04-12T23:02:37.614 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-04-12T23:02:38.286 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-12T23:02:38.295 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:02:38.295 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-04-12T23:02:38.322 INFO:teuthology.orchestra.run.smithi144.stderr:+ mkdir /mnt/foo 2024-04-12T23:02:38.364 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-04-12T23:02:38.430 INFO:teuthology.orchestra.run.smithi144.stderr:++ hostname 2024-04-12T23:02:38.431 INFO:teuthology.orchestra.run.smithi144.stderr:+ mount -t nfs smithi144:/fake /mnt/foo -o sync 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:38 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.491 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:38 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:38.850 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-04-12T23:02:38.877 INFO:teuthology.orchestra.run.smithi144.stderr:+ echo test 2024-04-12T23:02:38.904 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-04-12T23:02:38.970 INFO:teuthology.orchestra.run.smithi144.stderr:+ sync 2024-04-12T23:02:39.162 INFO:teuthology.run_tasks:Running task parallel... 2024-04-12T23:02:39.172 INFO:teuthology.task.parallel:starting parallel... 2024-04-12T23:02:39.173 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T23:02:39.173 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-12T23:02:39.173 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:02:39.174 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-12T23:02:39.174 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-12T23:02:39.174 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-12T23:02:39.175 INFO:teuthology.task.exec:Executing custom commands... 2024-04-12T23:02:39.175 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:02:39.175 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-04-12T23:02:39.272 INFO:teuthology.orchestra.run.smithi144.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-12T23:02:39.272 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:02:39.272 INFO:teuthology.orchestra.run.smithi144.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-04-12T23:02:39.273 INFO:teuthology.orchestra.run.smithi144.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-12T23:02:39.373 INFO:teuthology.orchestra.run.smithi144.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-12T23:02:39.374 INFO:teuthology.orchestra.run.smithi144.stdout:releasing clients 2024-04-12T23:02:39.412 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:39 smithi144 ceph-mon[24815]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:39.413 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:39 smithi144 ceph-mon[24815]: pgmap v230: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T23:02:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:39 smithi181 ceph-mon[26930]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:02:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:39 smithi181 ceph-mon[26930]: pgmap v230: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T23:02:40.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 295 54.43 MB/sec warmup 1 sec latency 11.479 ms 2024-04-12T23:02:40.950 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:40 smithi144 ceph-mon[24815]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T23:02:40.951 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:40 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:41.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:40 smithi181 ceph-mon[26930]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-04-12T23:02:41.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:40 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:02:41.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 597 51.56 MB/sec warmup 2 sec latency 12.901 ms 2024-04-12T23:02:42.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 1414 43.36 MB/sec warmup 3 sec latency 13.539 ms 2024-04-12T23:02:43.029 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:42 smithi144 ceph-mon[24815]: pgmap v232: 97 pgs: 97 active+clean; 41 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.5 MiB/s wr, 191 op/s 2024-04-12T23:02:43.166 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-12T23:02:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:42 smithi181 ceph-mon[26930]: pgmap v232: 97 pgs: 97 active+clean; 41 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.5 MiB/s wr, 191 op/s 2024-04-12T23:02:43.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 2486 35.85 MB/sec warmup 4 sec latency 9.708 ms 2024-04-12T23:02:44.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 3699 38.75 MB/sec warmup 5 sec latency 10.026 ms 2024-04-12T23:02:44.797 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:44 smithi144 ceph-mon[24815]: pgmap v233: 97 pgs: 97 active+clean; 41 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.5 MiB/s wr, 188 op/s 2024-04-12T23:02:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:44 smithi181 ceph-mon[26930]: pgmap v233: 97 pgs: 97 active+clean; 41 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.5 MiB/s wr, 188 op/s 2024-04-12T23:02:45.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 4158 33.50 MB/sec warmup 6 sec latency 198.153 ms 2024-04-12T23:02:46.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 5240 32.40 MB/sec warmup 7 sec latency 10.341 ms 2024-04-12T23:02:46.806 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:46 smithi144 ceph-mon[24815]: pgmap v234: 97 pgs: 97 active+clean; 123 MiB data, 179 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 12 MiB/s wr, 645 op/s 2024-04-12T23:02:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:46 smithi181 ceph-mon[26930]: pgmap v234: 97 pgs: 97 active+clean; 123 MiB data, 179 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 12 MiB/s wr, 645 op/s 2024-04-12T23:02:47.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 6111 30.30 MB/sec warmup 8 sec latency 12.173 ms 2024-04-12T23:02:48.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 7117 31.48 MB/sec warmup 9 sec latency 17.585 ms 2024-04-12T23:02:48.415 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-12T23:02:48.813 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:48 smithi144 ceph-mon[24815]: pgmap v235: 97 pgs: 97 active+clean; 132 MiB data, 216 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 13 MiB/s wr, 734 op/s 2024-04-12T23:02:49.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:48 smithi181 ceph-mon[26930]: pgmap v235: 97 pgs: 97 active+clean; 132 MiB data, 216 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 13 MiB/s wr, 734 op/s 2024-04-12T23:02:49.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 7846 29.86 MB/sec warmup 10 sec latency 18.398 ms 2024-04-12T23:02:49.818 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-12T23:02:49.818 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-12T23:02:49.818 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-04-12T23:02:49.818 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-12T23:02:49.818 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:02:49.818 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:49 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:02:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-12T23:02:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-12T23:02:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2024-04-12T23:02:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-12T23:02:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:02:50.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:49 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:02:50.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 8818 29.32 MB/sec warmup 11 sec latency 28.082 ms 2024-04-12T23:02:50.823 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:50 smithi144 ceph-mon[24815]: pgmap v236: 97 pgs: 97 active+clean; 134 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 15 MiB/s wr, 879 op/s 2024-04-12T23:02:50.824 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-12T23:02:50.824 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-12T23:02:50.824 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-04-12T23:02:50.824 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-12T23:02:50.824 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:50 smithi144 ceph-mon[24815]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T23:02:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:50 smithi181 ceph-mon[26930]: pgmap v236: 97 pgs: 97 active+clean; 134 MiB data, 260 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 15 MiB/s wr, 879 op/s 2024-04-12T23:02:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-12T23:02:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-12T23:02:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]': finished 2024-04-12T23:02:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-12T23:02:51.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:50 smithi181 ceph-mon[26930]: osdmap e69: 8 total, 8 up, 8 in 2024-04-12T23:02:51.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 9817 28.70 MB/sec warmup 12 sec latency 20.132 ms 2024-04-12T23:02:51.928 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:51 smithi144 ceph-mon[24815]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T23:02:52.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:51 smithi181 ceph-mon[26930]: osdmap e70: 8 total, 8 up, 8 in 2024-04-12T23:02:52.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 10779 29.69 MB/sec warmup 13 sec latency 19.816 ms 2024-04-12T23:02:52.932 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:52 smithi144 ceph-mon[24815]: pgmap v239: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 143 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 21 MiB/s wr, 1.41k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T23:02:52.932 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:52 smithi144 ceph-mon[24815]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T23:02:53.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:52 smithi181 ceph-mon[26930]: pgmap v239: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 143 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 21 MiB/s wr, 1.41k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T23:02:53.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:52 smithi181 ceph-mon[26930]: osdmap e71: 8 total, 8 up, 8 in 2024-04-12T23:02:53.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 11434 28.18 MB/sec warmup 14 sec latency 28.563 ms 2024-04-12T23:02:53.722 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph mgr module enable nfs --force' 2024-04-12T23:02:53.939 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:53 smithi144 ceph-mon[24815]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T23:02:54.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:53 smithi181 ceph-mon[26930]: osdmap e72: 8 total, 8 up, 8 in 2024-04-12T23:02:54.374 INFO:teuthology.orchestra.run.smithi144.stdout: 5 12180 27.45 MB/sec warmup 15 sec latency 20.184 ms 2024-04-12T23:02:54.950 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:54 smithi144 ceph-mon[24815]: pgmap v242: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 143 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 8.2 MiB/s wr, 734 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T23:02:54.951 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:54 smithi144 ceph-mon[24815]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T23:02:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:54 smithi181 ceph-mon[26930]: pgmap v242: 97 pgs: 1 remapped+peering, 1 peering, 95 active+clean; 143 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 8.2 MiB/s wr, 734 op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T23:02:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:54 smithi181 ceph-mon[26930]: osdmap e73: 8 total, 8 up, 8 in 2024-04-12T23:02:55.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 12973 26.56 MB/sec warmup 16 sec latency 34.876 ms 2024-04-12T23:02:56.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 13540 26.27 MB/sec warmup 17 sec latency 26.270 ms 2024-04-12T23:02:56.973 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:56 smithi144 ceph-mon[24815]: pgmap v244: 97 pgs: 1 remapped+peering, 96 active+clean; 158 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 413 B/s rd, 13 MiB/s wr, 1.38k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T23:02:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:56 smithi181 ceph-mon[26930]: pgmap v244: 97 pgs: 1 remapped+peering, 96 active+clean; 158 MiB data, 426 MiB used, 715 GiB / 715 GiB avail; 413 B/s rd, 13 MiB/s wr, 1.38k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-04-12T23:02:57.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 14097 26.12 MB/sec warmup 18 sec latency 34.322 ms 2024-04-12T23:02:57.876 INFO:teuthology.orchestra.run.smithi144.stderr:module 'nfs' is already enabled 2024-04-12T23:02:57.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:57 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3513546098' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T23:02:57.982 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:57 smithi144 ceph-mon[24815]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T23:02:58.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:57 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3513546098' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T23:02:58.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:57 smithi181 ceph-mon[26930]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-04-12T23:02:58.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 14473 25.52 MB/sec warmup 19 sec latency 41.276 ms 2024-04-12T23:02:58.981 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-12T23:02:59.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:58 smithi144 ceph-mon[24815]: pgmap v245: 97 pgs: 97 active+clean; 155 MiB data, 442 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.06k op/s; 683 KiB/s, 0 objects/s recovering 2024-04-12T23:02:59.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:58 smithi144 ceph-mon[24815]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-12T23:02:59.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:02:58 smithi144 ceph-mon[24815]: mgrmap e21: smithi144.rlueky(active, since 6m), standbys: smithi181.whsrmz 2024-04-12T23:02:59.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:58 smithi181 ceph-mon[26930]: pgmap v245: 97 pgs: 97 active+clean; 155 MiB data, 442 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.06k op/s; 683 KiB/s, 0 objects/s recovering 2024-04-12T23:02:59.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:58 smithi181 ceph-mon[26930]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-04-12T23:02:59.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:02:58 smithi181 ceph-mon[26930]: mgrmap e21: smithi144.rlueky(active, since 6m), standbys: smithi181.whsrmz 2024-04-12T23:02:59.375 INFO:teuthology.orchestra.run.smithi144.stdout: 5 14859 24.43 MB/sec warmup 20 sec latency 44.046 ms 2024-04-12T23:03:00.380 INFO:teuthology.orchestra.run.smithi144.stdout: 5 15480 23.79 MB/sec warmup 21 sec latency 65.247 ms 2024-04-12T23:03:01.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:00 smithi144 ceph-mon[24815]: pgmap v246: 97 pgs: 97 active+clean; 162 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.06k op/s; 516 KiB/s, 0 objects/s recovering 2024-04-12T23:03:01.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:00 smithi181 ceph-mon[26930]: pgmap v246: 97 pgs: 97 active+clean; 162 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 1.06k op/s; 516 KiB/s, 0 objects/s recovering 2024-04-12T23:03:01.380 INFO:teuthology.orchestra.run.smithi144.stdout: 5 16458 23.62 MB/sec warmup 22 sec latency 26.007 ms 2024-04-12T23:03:02.380 INFO:teuthology.orchestra.run.smithi144.stdout: 5 17110 23.62 MB/sec warmup 23 sec latency 35.602 ms 2024-04-12T23:03:02.618 INFO:teuthology.orchestra.run.smithi144.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T23:03:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:02 smithi181 ceph-mon[26930]: pgmap v247: 97 pgs: 97 active+clean; 166 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.08k op/s; 459 KiB/s, 0 objects/s recovering 2024-04-12T23:03:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:03:02.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:03:02.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:02 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:03:03.035 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:02 smithi144 ceph-mon[24815]: pgmap v247: 97 pgs: 97 active+clean; 166 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 12 MiB/s wr, 1.08k op/s; 459 KiB/s, 0 objects/s recovering 2024-04-12T23:03:03.035 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:03.035 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:03:03.036 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:03:03.036 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:02 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:03:03.380 INFO:teuthology.orchestra.run.smithi144.stdout: 5 17614 23.44 MB/sec warmup 24 sec latency 35.273 ms 2024-04-12T23:03:03.636 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-12T23:03:03.637 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:03:03.637 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-12T23:03:04.052 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:03 smithi144 ceph-mon[24815]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:04.052 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:03 smithi144 ceph-mon[24815]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T23:03:04.052 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:04.053 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:03 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:03 smithi181 ceph-mon[26930]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:03 smithi181 ceph-mon[26930]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T23:03:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:03 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:04.380 INFO:teuthology.orchestra.run.smithi144.stdout: 5 18151 23.30 MB/sec warmup 25 sec latency 42.128 ms 2024-04-12T23:03:05.066 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:04 smithi144 ceph-mon[24815]: pgmap v248: 97 pgs: 97 active+clean; 166 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 965 op/s; 409 KiB/s, 0 objects/s recovering 2024-04-12T23:03:05.066 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:04 smithi144 ceph-mon[24815]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T23:03:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:04 smithi181 ceph-mon[26930]: pgmap v248: 97 pgs: 97 active+clean; 166 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 965 op/s; 409 KiB/s, 0 objects/s recovering 2024-04-12T23:03:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:04 smithi181 ceph-mon[26930]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a 2024-04-12T23:03:05.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 18682 22.69 MB/sec warmup 26 sec latency 31.425 ms 2024-04-12T23:03:06.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 19307 22.44 MB/sec warmup 27 sec latency 50.769 ms 2024-04-12T23:03:07.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:06 smithi144 ceph-mon[24815]: pgmap v249: 97 pgs: 97 active+clean; 178 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 928 op/s; 343 KiB/s, 0 objects/s recovering 2024-04-12T23:03:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:06 smithi181 ceph-mon[26930]: pgmap v249: 97 pgs: 97 active+clean; 178 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 928 op/s; 343 KiB/s, 0 objects/s recovering 2024-04-12T23:03:07.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 19966 22.01 MB/sec warmup 28 sec latency 32.368 ms 2024-04-12T23:03:07.687 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:03:08.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 20686 22.13 MB/sec warmup 29 sec latency 26.456 ms 2024-04-12T23:03:08.688 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:03:08.688 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (5m) 31s ago 6m 22.4M - ba2b418f427c 5fe9fb07ff47 2024-04-12T23:03:08.688 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (6m) 31s ago 6m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:03:08.688 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (6m) 33s ago 6m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:03:08.688 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (5m) 31s ago 6m 54.7M - 8.3.5 dad864ee21e9 79755181f90b 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (61s) 31s ago 60s 14.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (58s) 33s ago 58s 12.1M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (8m) 31s ago 8m 459M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (6m) 33s ago 6m 411M - 17.2.0 e1d6a67b021e bd511fd5a495 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (8m) 31s ago 8m 47.9M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (6m) 33s ago 5m 38.2M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (42s) 31s ago 41s 47.0M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (36s) 33s ago 36s 48.6M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (6m) 31s ago 6m 17.6M - 1dbe0e931976 7410a549b2f7 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (5m) 33s ago 5m 20.3M - 1dbe0e931976 0d304d76f102 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (5m) 31s ago 5m 43.7M 3216M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (4m) 31s ago 4m 44.4M 3216M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (4m) 31s ago 4m 40.4M 3216M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (4m) 31s ago 4m 39.4M 3216M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:03:08.689 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (3m) 33s ago 3m 40.3M 3984M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:03:08.690 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (3m) 33s ago 3m 39.6M 3984M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:03:08.690 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (2m) 33s ago 2m 41.3M 3984M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:03:08.690 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (2m) 33s ago 2m 41.2M 3984M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:03:08.690 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (5m) 31s ago 5m 61.9M - 514e6a882f6e 3c5300d666ec 2024-04-12T23:03:08.942 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:08 smithi144 ceph-mon[24815]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:08.942 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:08 smithi144 ceph-mon[24815]: pgmap v250: 97 pgs: 97 active+clean; 179 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.1 MiB/s wr, 696 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-12T23:03:08.942 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:08 smithi144 ceph-mon[24815]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:09.136 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:08 smithi181 ceph-mon[26930]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:09.136 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:08 smithi181 ceph-mon[26930]: pgmap v250: 97 pgs: 97 active+clean; 179 MiB data, 621 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 8.1 MiB/s wr, 696 op/s; 341 KiB/s, 0 objects/s recovering 2024-04-12T23:03:09.136 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:08 smithi181 ceph-mon[26930]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:03:09.298 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:03:09.299 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:03:09.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 21428 22.32 MB/sec warmup 30 sec latency 23.217 ms 2024-04-12T23:03:09.785 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:03:09.786 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", 2024-04-12T23:03:09.786 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:03:09.786 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:03:09.786 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "", 2024-04-12T23:03:09.786 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a image" 2024-04-12T23:03:09.786 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:03:09.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:09 smithi144 ceph-mon[24815]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:09.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:09 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2578285845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:03:10.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:09 smithi181 ceph-mon[26930]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:10.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:09 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2578285845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:03:10.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 21867 21.92 MB/sec warmup 31 sec latency 38.218 ms 2024-04-12T23:03:10.387 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:03:10.964 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:10 smithi144 ceph-mon[24815]: pgmap v251: 97 pgs: 97 active+clean; 178 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.7 MiB/s wr, 731 op/s 2024-04-12T23:03:10.964 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:10 smithi144 ceph-mon[24815]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:10.965 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:10 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2073697921' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:03:11.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:10 smithi181 ceph-mon[26930]: pgmap v251: 97 pgs: 97 active+clean; 178 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.7 MiB/s wr, 731 op/s 2024-04-12T23:03:11.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:10 smithi181 ceph-mon[26930]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:11.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:10 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2073697921' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:03:11.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 22405 21.48 MB/sec warmup 32 sec latency 21.619 ms 2024-04-12T23:03:12.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 22929 21.30 MB/sec warmup 33 sec latency 26.464 ms 2024-04-12T23:03:12.979 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:12 smithi144 ceph-mon[24815]: pgmap v252: 97 pgs: 97 active+clean; 190 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 890 op/s 2024-04-12T23:03:13.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:12 smithi181 ceph-mon[26930]: pgmap v252: 97 pgs: 97 active+clean; 190 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 890 op/s 2024-04-12T23:03:13.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 23556 20.95 MB/sec warmup 34 sec latency 28.307 ms 2024-04-12T23:03:14.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 24142 20.89 MB/sec warmup 35 sec latency 28.599 ms 2024-04-12T23:03:14.989 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:14 smithi144 ceph-mon[24815]: pgmap v253: 97 pgs: 97 active+clean; 190 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 786 op/s 2024-04-12T23:03:15.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:14 smithi181 ceph-mon[26930]: pgmap v253: 97 pgs: 97 active+clean; 190 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 786 op/s 2024-04-12T23:03:15.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 24897 21.21 MB/sec warmup 36 sec latency 23.940 ms 2024-04-12T23:03:16.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 25485 20.98 MB/sec warmup 37 sec latency 19.401 ms 2024-04-12T23:03:17.001 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:16 smithi144 ceph-mon[24815]: pgmap v254: 97 pgs: 97 active+clean; 191 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 960 op/s 2024-04-12T23:03:17.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:16 smithi181 ceph-mon[26930]: pgmap v254: 97 pgs: 97 active+clean; 191 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 960 op/s 2024-04-12T23:03:17.381 INFO:teuthology.orchestra.run.smithi144.stdout: 5 26262 20.82 MB/sec warmup 38 sec latency 17.584 ms 2024-04-12T23:03:18.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 27085 20.71 MB/sec warmup 39 sec latency 31.968 ms 2024-04-12T23:03:19.021 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:18 smithi144 ceph-mon[24815]: pgmap v255: 97 pgs: 97 active+clean; 192 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.4 MiB/s wr, 653 op/s 2024-04-12T23:03:19.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:18 smithi181 ceph-mon[26930]: pgmap v255: 97 pgs: 97 active+clean; 192 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.4 MiB/s wr, 653 op/s 2024-04-12T23:03:19.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 27779 20.85 MB/sec warmup 40 sec latency 28.556 ms 2024-04-12T23:03:20.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 28544 21.01 MB/sec warmup 41 sec latency 20.596 ms 2024-04-12T23:03:21.035 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:20 smithi144 ceph-mon[24815]: pgmap v256: 97 pgs: 97 active+clean; 197 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 793 op/s 2024-04-12T23:03:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:20 smithi181 ceph-mon[26930]: pgmap v256: 97 pgs: 97 active+clean; 197 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 793 op/s 2024-04-12T23:03:21.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 29000 20.75 MB/sec warmup 42 sec latency 32.971 ms 2024-04-12T23:03:22.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 29732 20.53 MB/sec warmup 43 sec latency 21.584 ms 2024-04-12T23:03:23.051 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:22 smithi144 ceph-mon[24815]: pgmap v257: 97 pgs: 97 active+clean; 208 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.06k op/s 2024-04-12T23:03:23.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:22 smithi181 ceph-mon[26930]: pgmap v257: 97 pgs: 97 active+clean; 208 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.06k op/s 2024-04-12T23:03:23.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 30625 20.54 MB/sec warmup 44 sec latency 35.454 ms 2024-04-12T23:03:24.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 31084 20.32 MB/sec warmup 45 sec latency 23.763 ms 2024-04-12T23:03:24.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:24 smithi144 ceph-mon[24815]: pgmap v258: 97 pgs: 97 active+clean; 208 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-04-12T23:03:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:24 smithi181 ceph-mon[26930]: pgmap v258: 97 pgs: 97 active+clean; 208 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 738 op/s 2024-04-12T23:03:25.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 31811 20.55 MB/sec warmup 46 sec latency 24.943 ms 2024-04-12T23:03:26.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 32393 20.53 MB/sec warmup 47 sec latency 19.364 ms 2024-04-12T23:03:26.967 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:26 smithi144 ceph-mon[24815]: pgmap v259: 97 pgs: 97 active+clean; 210 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 950 op/s 2024-04-12T23:03:27.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:26 smithi181 ceph-mon[26930]: pgmap v259: 97 pgs: 97 active+clean; 210 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 950 op/s 2024-04-12T23:03:27.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 33154 20.35 MB/sec warmup 48 sec latency 19.478 ms 2024-04-12T23:03:28.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 34024 20.31 MB/sec warmup 49 sec latency 22.464 ms 2024-04-12T23:03:28.986 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:28 smithi144 ceph-mon[24815]: pgmap v260: 97 pgs: 97 active+clean; 215 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 906 op/s 2024-04-12T23:03:29.140 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:28 smithi181 ceph-mon[26930]: pgmap v260: 97 pgs: 97 active+clean; 215 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 906 op/s 2024-04-12T23:03:29.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 34704 20.34 MB/sec warmup 50 sec latency 21.917 ms 2024-04-12T23:03:30.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 35382 20.42 MB/sec warmup 51 sec latency 22.981 ms 2024-04-12T23:03:31.011 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:30 smithi144 ceph-mon[24815]: pgmap v261: 97 pgs: 97 active+clean; 220 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 924 op/s 2024-04-12T23:03:31.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:30 smithi181 ceph-mon[26930]: pgmap v261: 97 pgs: 97 active+clean; 220 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 924 op/s 2024-04-12T23:03:31.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 35968 20.41 MB/sec warmup 52 sec latency 32.333 ms 2024-04-12T23:03:32.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 36591 20.19 MB/sec warmup 53 sec latency 16.914 ms 2024-04-12T23:03:33.029 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:32 smithi144 ceph-mon[24815]: pgmap v262: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-12T23:03:33.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:32 smithi181 ceph-mon[26930]: pgmap v262: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-12T23:03:33.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 37299 20.17 MB/sec warmup 54 sec latency 24.896 ms 2024-04-12T23:03:34.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 37955 19.98 MB/sec warmup 55 sec latency 35.383 ms 2024-04-12T23:03:34.794 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:34 smithi144 ceph-mon[24815]: pgmap v263: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 684 op/s 2024-04-12T23:03:35.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:34 smithi181 ceph-mon[26930]: pgmap v263: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 684 op/s 2024-04-12T23:03:35.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 38653 20.17 MB/sec warmup 56 sec latency 22.476 ms 2024-04-12T23:03:36.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 39281 20.21 MB/sec warmup 57 sec latency 25.556 ms 2024-04-12T23:03:36.810 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:36 smithi144 ceph-mon[24815]: pgmap v264: 97 pgs: 97 active+clean; 229 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 876 op/s 2024-04-12T23:03:37.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:36 smithi181 ceph-mon[26930]: pgmap v264: 97 pgs: 97 active+clean; 229 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 876 op/s 2024-04-12T23:03:37.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 39800 19.99 MB/sec warmup 58 sec latency 73.440 ms 2024-04-12T23:03:38.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 40538 19.89 MB/sec warmup 59 sec latency 23.104 ms 2024-04-12T23:03:38.827 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:38 smithi144 ceph-mon[24815]: pgmap v265: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.2 MiB/s wr, 901 op/s 2024-04-12T23:03:39.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:38 smithi181 ceph-mon[26930]: pgmap v265: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.2 MiB/s wr, 901 op/s 2024-04-12T23:03:39.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 41328 19.87 MB/sec warmup 60 sec latency 47.663 ms 2024-04-12T23:03:40.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 42012 19.91 MB/sec warmup 61 sec latency 72.822 ms 2024-04-12T23:03:40.911 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:40 smithi144 ceph-mon[24815]: pgmap v266: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 822 op/s 2024-04-12T23:03:40.936 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:03:41.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:40 smithi181 ceph-mon[26930]: pgmap v266: 97 pgs: 97 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 822 op/s 2024-04-12T23:03:41.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 42776 20.11 MB/sec warmup 62 sec latency 51.464 ms 2024-04-12T23:03:41.915 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:03:41.916 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (5m) 64s ago 7m 22.4M - ba2b418f427c 5fe9fb07ff47 2024-04-12T23:03:41.916 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (7m) 64s ago 7m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:03:41.916 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (6m) 66s ago 6m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:03:41.916 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (5m) 64s ago 7m 54.7M - 8.3.5 dad864ee21e9 79755181f90b 2024-04-12T23:03:41.916 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (95s) 64s ago 94s 14.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:03:41.916 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (91s) 66s ago 91s 12.1M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (8m) 64s ago 8m 459M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (6m) 66s ago 6m 411M - 17.2.0 e1d6a67b021e bd511fd5a495 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (8m) 64s ago 8m 47.9M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (6m) 66s ago 6m 38.2M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (75s) 64s ago 75s 47.0M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (70s) 66s ago 69s 48.6M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:03:41.917 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 64s ago 6m 17.6M - 1dbe0e931976 7410a549b2f7 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 66s ago 6m 20.3M - 1dbe0e931976 0d304d76f102 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (5m) 64s ago 5m 43.7M 3216M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (5m) 64s ago 5m 44.4M 3216M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (4m) 64s ago 4m 40.4M 3216M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (4m) 64s ago 4m 39.4M 3216M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (4m) 66s ago 4m 40.3M 3984M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:03:41.918 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (3m) 66s ago 3m 39.6M 3984M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:03:41.919 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (3m) 66s ago 3m 41.3M 3984M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:03:41.919 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (2m) 66s ago 2m 41.2M 3984M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:03:41.919 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (6m) 64s ago 6m 61.9M - 514e6a882f6e 3c5300d666ec 2024-04-12T23:03:41.920 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:41 smithi144 ceph-mon[24815]: from='client.24375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:41 smithi181 ceph-mon[26930]: from='client.24375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:42.383 INFO:teuthology.orchestra.run.smithi144.stdout: 5 43413 19.97 MB/sec warmup 63 sec latency 77.559 ms 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:03:42.548 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:03:42.549 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-12T23:03:42.550 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:03:42.550 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:03:42.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:42 smithi144 ceph-mon[24815]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:42.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:42 smithi144 ceph-mon[24815]: pgmap v267: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-12T23:03:42.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:42 smithi144 ceph-mon[24815]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:42.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:42 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/3156005215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:03:43.077 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:03:43.077 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a", 2024-04-12T23:03:43.077 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:03:43.077 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:03:43.077 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "", 2024-04-12T23:03:43.077 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:b277feb2c7beb4597d8ecc39d58d5c36b81f613a image" 2024-04-12T23:03:43.078 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:03:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:42 smithi181 ceph-mon[26930]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:42 smithi181 ceph-mon[26930]: pgmap v267: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-04-12T23:03:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:42 smithi181 ceph-mon[26930]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:42 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3156005215' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:03:43.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 44178 19.90 MB/sec warmup 64 sec latency 19.806 ms 2024-04-12T23:03:43.708 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:03:43.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:43 smithi144 ceph-mon[24815]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:43.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:43 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2054697764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:03:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:43 smithi181 ceph-mon[26930]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:03:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:43 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2054697764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:03:44.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 44863 19.80 MB/sec warmup 65 sec latency 32.652 ms 2024-04-12T23:03:44.835 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:44 smithi144 ceph-mon[24815]: pgmap v268: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 782 op/s 2024-04-12T23:03:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:44 smithi181 ceph-mon[26930]: pgmap v268: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 782 op/s 2024-04-12T23:03:45.383 INFO:teuthology.orchestra.run.smithi144.stdout: 5 45209 19.63 MB/sec warmup 66 sec latency 76.711 ms 2024-04-12T23:03:46.383 INFO:teuthology.orchestra.run.smithi144.stdout: 5 45740 19.72 MB/sec warmup 67 sec latency 33.684 ms 2024-04-12T23:03:47.127 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: pgmap v269: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1k op/s 2024-04-12T23:03:47.128 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:47.128 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T23:03:47.128 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T23:03:47.128 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:03:47.128 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:47.128 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:47 smithi144 ceph-mon[24815]: Upgrade: Need to upgrade myself (mgr.smithi144.rlueky) 2024-04-12T23:03:47.382 INFO:teuthology.orchestra.run.smithi144.stdout: 5 46348 19.75 MB/sec warmup 68 sec latency 22.416 ms 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: pgmap v269: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1k op/s 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: Upgrade: Target is version 19.0.0-2053-gb277feb2 (unknown) 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec'] 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:03:47.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:47 smithi181 ceph-mon[26930]: Upgrade: Need to upgrade myself (mgr.smithi144.rlueky) 2024-04-12T23:03:48.152 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:48 smithi144 ceph-mon[24815]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi181 2024-04-12T23:03:48.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 46725 19.56 MB/sec warmup 69 sec latency 30.429 ms 2024-04-12T23:03:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:48 smithi181 ceph-mon[26930]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec on smithi181 2024-04-12T23:03:49.163 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:49 smithi144 ceph-mon[24815]: pgmap v270: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 911 op/s 2024-04-12T23:03:49.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 47443 19.46 MB/sec warmup 70 sec latency 18.506 ms 2024-04-12T23:03:49.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:49 smithi181 ceph-mon[26930]: pgmap v270: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 911 op/s 2024-04-12T23:03:50.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:03:50.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:50 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:03:50.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 48190 19.40 MB/sec warmup 71 sec latency 43.491 ms 2024-04-12T23:03:50.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:03:50.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:50 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:03:51.176 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:51 smithi144 ceph-mon[24815]: pgmap v271: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 860 op/s 2024-04-12T23:03:51.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 48910 19.43 MB/sec warmup 72 sec latency 22.958 ms 2024-04-12T23:03:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:51 smithi181 ceph-mon[26930]: pgmap v271: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 860 op/s 2024-04-12T23:03:52.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 49564 19.51 MB/sec warmup 73 sec latency 30.039 ms 2024-04-12T23:03:53.190 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:53 smithi144 ceph-mon[24815]: pgmap v272: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-12T23:03:53.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 50097 19.50 MB/sec warmup 74 sec latency 28.235 ms 2024-04-12T23:03:53.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:53 smithi181 ceph-mon[26930]: pgmap v272: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-04-12T23:03:54.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 50629 19.33 MB/sec warmup 75 sec latency 26.007 ms 2024-04-12T23:03:54.957 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:54 smithi144 ceph-mon[24815]: pgmap v273: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 703 op/s 2024-04-12T23:03:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:54 smithi181 ceph-mon[26930]: pgmap v273: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 703 op/s 2024-04-12T23:03:55.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 51189 19.23 MB/sec warmup 76 sec latency 28.183 ms 2024-04-12T23:03:56.390 INFO:teuthology.orchestra.run.smithi144.stdout: 5 52163 19.25 MB/sec warmup 77 sec latency 20.284 ms 2024-04-12T23:03:56.981 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:56 smithi144 ceph-mon[24815]: pgmap v274: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-12T23:03:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:56 smithi181 ceph-mon[26930]: pgmap v274: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-04-12T23:03:57.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 52934 19.42 MB/sec warmup 78 sec latency 25.929 ms 2024-04-12T23:03:58.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 53638 19.48 MB/sec warmup 79 sec latency 24.537 ms 2024-04-12T23:03:58.998 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:03:58 smithi144 ceph-mon[24815]: pgmap v275: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 856 op/s 2024-04-12T23:03:59.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:03:58 smithi181 ceph-mon[26930]: pgmap v275: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 856 op/s 2024-04-12T23:03:59.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 54288 19.33 MB/sec warmup 80 sec latency 29.053 ms 2024-04-12T23:04:00.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 55039 19.35 MB/sec warmup 81 sec latency 19.853 ms 2024-04-12T23:04:01.072 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:00 smithi144 ceph-mon[24815]: pgmap v276: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.6 MiB/s wr, 907 op/s 2024-04-12T23:04:01.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:00 smithi181 ceph-mon[26930]: pgmap v276: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.6 MiB/s wr, 907 op/s 2024-04-12T23:04:01.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 55688 19.24 MB/sec warmup 82 sec latency 276.679 ms 2024-04-12T23:04:02.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 56004 19.16 MB/sec warmup 83 sec latency 730.551 ms 2024-04-12T23:04:02.843 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:02 smithi144 ceph-mon[24815]: pgmap v277: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 976 op/s 2024-04-12T23:04:03.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:02 smithi181 ceph-mon[26930]: pgmap v277: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 976 op/s 2024-04-12T23:04:03.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 56833 19.28 MB/sec warmup 84 sec latency 23.540 ms 2024-04-12T23:04:04.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 57307 19.24 MB/sec warmup 85 sec latency 25.181 ms 2024-04-12T23:04:04.859 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:04 smithi144 ceph-mon[24815]: pgmap v278: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 780 op/s 2024-04-12T23:04:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:04 smithi181 ceph-mon[26930]: pgmap v278: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 780 op/s 2024-04-12T23:04:05.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 57821 19.09 MB/sec warmup 86 sec latency 50.639 ms 2024-04-12T23:04:06.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 58296 19.00 MB/sec warmup 87 sec latency 24.444 ms 2024-04-12T23:04:06.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:06 smithi144 ceph-mon[24815]: pgmap v279: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 979 op/s 2024-04-12T23:04:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:06 smithi181 ceph-mon[26930]: pgmap v279: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 979 op/s 2024-04-12T23:04:07.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 58964 18.98 MB/sec warmup 88 sec latency 32.331 ms 2024-04-12T23:04:08.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 59458 18.89 MB/sec warmup 89 sec latency 50.276 ms 2024-04-12T23:04:08.942 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:08 smithi144 ceph-mon[24815]: pgmap v280: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.4 MiB/s wr, 764 op/s 2024-04-12T23:04:09.135 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:08 smithi181 ceph-mon[26930]: pgmap v280: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.4 MiB/s wr, 764 op/s 2024-04-12T23:04:09.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 59759 18.84 MB/sec warmup 90 sec latency 46.080 ms 2024-04-12T23:04:10.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 60438 18.89 MB/sec warmup 91 sec latency 36.987 ms 2024-04-12T23:04:10.969 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:10 smithi144 ceph-mon[24815]: pgmap v281: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.1 MiB/s wr, 836 op/s 2024-04-12T23:04:11.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:10 smithi181 ceph-mon[26930]: pgmap v281: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.1 MiB/s wr, 836 op/s 2024-04-12T23:04:11.389 INFO:teuthology.orchestra.run.smithi144.stdout: 5 60928 18.83 MB/sec warmup 92 sec latency 29.324 ms 2024-04-12T23:04:12.392 INFO:teuthology.orchestra.run.smithi144.stdout: 5 61415 18.69 MB/sec warmup 93 sec latency 35.411 ms 2024-04-12T23:04:12.998 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:12 smithi144 ceph-mon[24815]: pgmap v282: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 845 op/s 2024-04-12T23:04:13.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:12 smithi181 ceph-mon[26930]: pgmap v282: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.9 MiB/s wr, 845 op/s 2024-04-12T23:04:13.392 INFO:teuthology.orchestra.run.smithi144.stdout: 5 61868 18.61 MB/sec warmup 94 sec latency 26.650 ms 2024-04-12T23:04:14.197 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:04:14.392 INFO:teuthology.orchestra.run.smithi144.stdout: 5 62529 18.59 MB/sec warmup 95 sec latency 68.529 ms 2024-04-12T23:04:15.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:14 smithi144 ceph-mon[24815]: pgmap v283: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.4 MiB/s wr, 590 op/s 2024-04-12T23:04:15.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:14 smithi144 ceph-mon[24815]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (6m) 97s ago 7m 22.4M - ba2b418f427c 5fe9fb07ff47 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (7m) 97s ago 7m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (7m) 100s ago 7m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (6m) 97s ago 7m 54.7M - 8.3.5 dad864ee21e9 79755181f90b 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (2m) 97s ago 2m 14.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (2m) 100s ago 2m 12.1M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (9m) 97s ago 9m 459M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (7m) 100s ago 7m 411M - 17.2.0 e1d6a67b021e bd511fd5a495 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (9m) 97s ago 9m 47.9M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (7m) 100s ago 7m 38.2M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (108s) 97s ago 108s 47.0M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:04:15.044 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (103s) 100s ago 102s 48.6M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 97s ago 7m 17.6M - 1dbe0e931976 7410a549b2f7 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 100s ago 6m 20.3M - 1dbe0e931976 0d304d76f102 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (6m) 97s ago 6m 43.7M 3216M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (5m) 97s ago 5m 44.4M 3216M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (5m) 97s ago 5m 40.4M 3216M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (5m) 97s ago 5m 39.4M 3216M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (4m) 100s ago 4m 40.3M 3984M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (4m) 100s ago 4m 39.6M 3984M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (3m) 100s ago 3m 41.3M 3984M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (3m) 100s ago 3m 41.2M 3984M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:04:15.045 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (6m) 97s ago 6m 61.9M - 514e6a882f6e 3c5300d666ec 2024-04-12T23:04:15.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:14 smithi181 ceph-mon[26930]: pgmap v283: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.4 MiB/s wr, 590 op/s 2024-04-12T23:04:15.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:14 smithi181 ceph-mon[26930]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:15.392 INFO:teuthology.orchestra.run.smithi144.stdout: 5 63011 18.51 MB/sec warmup 96 sec latency 59.358 ms 2024-04-12T23:04:15.559 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:04:15.560 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:04:15.922 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:04:15.922 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:04:15.922 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:04:15.922 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:04:15.922 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "0/23 daemons upgraded", 2024-04-12T23:04:15.923 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec image on host smithi181" 2024-04-12T23:04:15.923 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:04:16.049 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:15 smithi144 ceph-mon[24815]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:16.050 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:15 smithi144 ceph-mon[24815]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:16.050 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:15 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1610527102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:04:16.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:15 smithi181 ceph-mon[26930]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:16.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:15 smithi181 ceph-mon[26930]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:16.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:15 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1610527102' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:04:16.392 INFO:teuthology.orchestra.run.smithi144.stdout: 5 63461 18.53 MB/sec warmup 97 sec latency 83.591 ms 2024-04-12T23:04:16.476 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:04:16.803 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:16 smithi144 ceph-mon[24815]: pgmap v284: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.2 MiB/s wr, 888 op/s 2024-04-12T23:04:16.803 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:16 smithi144 ceph-mon[24815]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:16.803 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:16 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/536442458' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:04:17.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:16 smithi181 ceph-mon[26930]: pgmap v284: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.2 MiB/s wr, 888 op/s 2024-04-12T23:04:17.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:16 smithi181 ceph-mon[26930]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:17.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:16 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/536442458' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:04:17.392 INFO:teuthology.orchestra.run.smithi144.stdout: 5 64043 18.55 MB/sec warmup 98 sec latency 45.027 ms 2024-04-12T23:04:18.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 64465 18.46 MB/sec warmup 99 sec latency 27.430 ms 2024-04-12T23:04:18.836 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:18 smithi144 ceph-mon[24815]: pgmap v285: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.0 MiB/s wr, 712 op/s 2024-04-12T23:04:19.136 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:18 smithi181 ceph-mon[26930]: pgmap v285: 97 pgs: 97 active+clean; 315 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.0 MiB/s wr, 712 op/s 2024-04-12T23:04:19.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 65006 18.35 MB/sec warmup 100 sec latency 32.747 ms 2024-04-12T23:04:20.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 65463 18.27 MB/sec warmup 101 sec latency 21.959 ms 2024-04-12T23:04:20.609 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: Upgrade: Updating mgr.smithi181.whsrmz 2024-04-12T23:04:20.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:20.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:04:20.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:04:20.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:04:20.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: Deploying daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T23:04:20.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:20 smithi144 ceph-mon[24815]: pgmap v286: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 653 op/s 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: Upgrade: Updating mgr.smithi181.whsrmz 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: Deploying daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T23:04:20.911 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:20 smithi181 ceph-mon[26930]: pgmap v286: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 653 op/s 2024-04-12T23:04:21.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 66164 18.26 MB/sec warmup 102 sec latency 54.522 ms 2024-04-12T23:04:22.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 66669 18.23 MB/sec warmup 103 sec latency 31.058 ms 2024-04-12T23:04:22.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:22 smithi144 ceph-mon[24815]: pgmap v287: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 801 op/s 2024-04-12T23:04:22.969 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:22 smithi181 ceph-mon[26930]: pgmap v287: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 801 op/s 2024-04-12T23:04:23.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 67146 18.24 MB/sec warmup 104 sec latency 97.059 ms 2024-04-12T23:04:24.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 67735 18.26 MB/sec warmup 105 sec latency 25.694 ms 2024-04-12T23:04:24.730 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:24.730 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:04:24.731 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:04:24.731 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:24 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:04:24.731 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:24 smithi144 ceph-mon[24815]: pgmap v288: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.5 MiB/s wr, 638 op/s 2024-04-12T23:04:24.822 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:24.822 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:04:24.822 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:04:24.822 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:24 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:04:24.822 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:24 smithi181 ceph-mon[26930]: pgmap v288: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.5 MiB/s wr, 638 op/s 2024-04-12T23:04:25.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 68020 18.14 MB/sec warmup 106 sec latency 40.323 ms 2024-04-12T23:04:26.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 68449 18.02 MB/sec warmup 107 sec latency 43.542 ms 2024-04-12T23:04:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:26 smithi181 ceph-mon[26930]: pgmap v289: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.4 MiB/s wr, 949 op/s 2024-04-12T23:04:27.017 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:26 smithi144 ceph-mon[24815]: pgmap v289: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 9.4 MiB/s wr, 949 op/s 2024-04-12T23:04:27.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 68762 17.91 MB/sec warmup 108 sec latency 99.211 ms 2024-04-12T23:04:28.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 69261 17.92 MB/sec warmup 109 sec latency 43.469 ms 2024-04-12T23:04:29.303 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:29 smithi181 ceph-mon[26930]: pgmap v290: 97 pgs: 97 active+clean; 343 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 735 op/s 2024-04-12T23:04:29.303 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:29 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:29.303 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:29 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:29.304 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:29 smithi144 ceph-mon[24815]: pgmap v290: 97 pgs: 97 active+clean; 343 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.2 MiB/s wr, 735 op/s 2024-04-12T23:04:29.304 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:29 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:29.305 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:29 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:29.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 69730 17.81 MB/sec warmup 110 sec latency 50.734 ms 2024-04-12T23:04:30.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 70238 17.78 MB/sec warmup 111 sec latency 67.751 ms 2024-04-12T23:04:31.081 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:31 smithi144 ceph-mon[24815]: pgmap v291: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.5 MiB/s wr, 751 op/s 2024-04-12T23:04:31.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 70878 17.83 MB/sec warmup 112 sec latency 27.455 ms 2024-04-12T23:04:31.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:31 smithi181 ceph-mon[26930]: pgmap v291: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.5 MiB/s wr, 751 op/s 2024-04-12T23:04:32.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 71385 17.85 MB/sec warmup 113 sec latency 29.825 ms 2024-04-12T23:04:33.097 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: pgmap v292: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 932 op/s 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: Standby manager daemon smithi181.whsrmz restarted 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr fail", "who": "smithi144.rlueky"}]: dispatch 2024-04-12T23:04:33.098 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:33 smithi144 ceph-mon[24815]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T23:04:33.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: pgmap v292: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 932 op/s 2024-04-12T23:04:33.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz restarted 2024-04-12T23:04:33.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:04:33.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:04:33.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:04:33.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr fail", "who": "smithi144.rlueky"}]: dispatch 2024-04-12T23:04:33.318 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:33 smithi181 ceph-mon[26930]: osdmap e74: 8 total, 8 up, 8 in 2024-04-12T23:04:33.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 71786 17.74 MB/sec warmup 114 sec latency 47.608 ms 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "mgr fail", "who": "smithi144.rlueky"}]': finished 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: mgrmap e22: smithi181.whsrmz(active, starting, since 0.509798s) 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:04:34.104 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: Manager daemon smithi181.whsrmz is now available 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:04:34.105 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:04:34.106 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:04:34.106 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/trash_purge_schedule"}]: dispatch 2024-04-12T23:04:34.106 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/trash_purge_schedule"}]: dispatch 2024-04-12T23:04:34.106 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:34.106 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:34.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 72285 17.65 MB/sec warmup 115 sec latency 25.015 ms 2024-04-12T23:04:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.14204 172.21.15.144:0/3931199579' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "mgr fail", "who": "smithi144.rlueky"}]': finished 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: mgrmap e22: smithi181.whsrmz(active, starting, since 0.509798s) 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:04:34.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: Manager daemon smithi181.whsrmz is now available 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:04:34.450 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:04:34.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:04:34.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/trash_purge_schedule"}]: dispatch 2024-04-12T23:04:34.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi181.whsrmz/trash_purge_schedule"}]: dispatch 2024-04-12T23:04:34.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:34.451 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:35.111 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:35 smithi144 ceph-mon[24815]: Deploying cephadm binary to smithi181 2024-04-12T23:04:35.111 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:35 smithi144 ceph-mon[24815]: mgrmap e23: smithi181.whsrmz(active, since 1.5122s) 2024-04-12T23:04:35.112 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:35 smithi144 ceph-mon[24815]: pgmap v3: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-12T23:04:35.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 72809 17.66 MB/sec warmup 116 sec latency 33.596 ms 2024-04-12T23:04:35.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:35 smithi181 ceph-mon[26930]: Deploying cephadm binary to smithi181 2024-04-12T23:04:35.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:35 smithi181 ceph-mon[26930]: mgrmap e23: smithi181.whsrmz(active, since 1.5122s) 2024-04-12T23:04:35.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:35 smithi181 ceph-mon[26930]: pgmap v3: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-12T23:04:36.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:36 smithi144 ceph-mon[24815]: mgrmap e24: smithi181.whsrmz(active, since 2s) 2024-04-12T23:04:36.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:36 smithi144 ceph-mon[24815]: pgmap v4: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-12T23:04:36.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:36 smithi144 ceph-mon[24815]: [12/Apr/2024:23:04:35] ENGINE Bus STARTING 2024-04-12T23:04:36.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 73455 17.58 MB/sec warmup 117 sec latency 38.767 ms 2024-04-12T23:04:36.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:36 smithi181 ceph-mon[26930]: mgrmap e24: smithi181.whsrmz(active, since 2s) 2024-04-12T23:04:36.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:36 smithi181 ceph-mon[26930]: pgmap v4: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-12T23:04:36.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:36 smithi181 ceph-mon[26930]: [12/Apr/2024:23:04:35] ENGINE Bus STARTING 2024-04-12T23:04:37.135 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:37 smithi144 ceph-mon[24815]: [12/Apr/2024:23:04:35] ENGINE Serving on http://172.21.15.181:8765 2024-04-12T23:04:37.135 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:37 smithi144 ceph-mon[24815]: [12/Apr/2024:23:04:35] ENGINE Serving on https://172.21.15.181:7150 2024-04-12T23:04:37.135 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:37 smithi144 ceph-mon[24815]: [12/Apr/2024:23:04:35] ENGINE Bus STARTED 2024-04-12T23:04:37.393 INFO:teuthology.orchestra.run.smithi144.stdout: 5 73895 17.58 MB/sec warmup 118 sec latency 25.683 ms 2024-04-12T23:04:37.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:37 smithi181 ceph-mon[26930]: [12/Apr/2024:23:04:35] ENGINE Serving on http://172.21.15.181:8765 2024-04-12T23:04:37.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:37 smithi181 ceph-mon[26930]: [12/Apr/2024:23:04:35] ENGINE Serving on https://172.21.15.181:7150 2024-04-12T23:04:37.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:37 smithi181 ceph-mon[26930]: [12/Apr/2024:23:04:35] ENGINE Bus STARTED 2024-04-12T23:04:38.148 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:38 smithi144 ceph-mon[24815]: pgmap v5: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-12T23:04:38.148 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:38 smithi144 ceph-mon[24815]: mgrmap e25: smithi181.whsrmz(active, since 4s) 2024-04-12T23:04:38.148 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:38 smithi144 ceph-mon[24815]: Deploying cephadm binary to smithi144 2024-04-12T23:04:38.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 74329 17.56 MB/sec warmup 119 sec latency 48.614 ms 2024-04-12T23:04:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:38 smithi181 ceph-mon[26930]: pgmap v5: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-04-12T23:04:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:38 smithi181 ceph-mon[26930]: mgrmap e25: smithi181.whsrmz(active, since 4s) 2024-04-12T23:04:38.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:38 smithi181 ceph-mon[26930]: Deploying cephadm binary to smithi144 2024-04-12T23:04:39.915 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:39 smithi144 ceph-mon[24815]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 864 KiB/s wr, 68 op/s 2024-04-12T23:04:40.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:39 smithi181 ceph-mon[26930]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 864 KiB/s wr, 68 op/s 2024-04-12T23:04:40.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 75378 7.26 MB/sec execute 1 sec latency 29.016 ms 2024-04-12T23:04:41.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 75755 5.77 MB/sec execute 2 sec latency 43.355 ms 2024-04-12T23:04:42.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:42 smithi144 ceph-mon[24815]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 5.5 MiB/s wr, 521 op/s 2024-04-12T23:04:42.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:42 smithi144 ceph-mon[24815]: Standby manager daemon smithi144.rlueky started 2024-04-12T23:04:42.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:42 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/crt"}]: dispatch 2024-04-12T23:04:42.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:42 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:04:42.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:42 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/key"}]: dispatch 2024-04-12T23:04:42.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:42 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:04:42.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 76141 7.93 MB/sec execute 3 sec latency 49.515 ms 2024-04-12T23:04:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:42 smithi181 ceph-mon[26930]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 5.5 MiB/s wr, 521 op/s 2024-04-12T23:04:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:42 smithi181 ceph-mon[26930]: Standby manager daemon smithi144.rlueky started 2024-04-12T23:04:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:42 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/crt"}]: dispatch 2024-04-12T23:04:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:42 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:04:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:42 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/key"}]: dispatch 2024-04-12T23:04:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:42 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/3159061657' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:04:43.208 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:43 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T23:04:43.208 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:43 smithi144 ceph-mon[24815]: mgrmap e26: smithi181.whsrmz(active, since 9s), standbys: smithi144.rlueky 2024-04-12T23:04:43.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 76700 9.09 MB/sec execute 4 sec latency 87.995 ms 2024-04-12T23:04:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:43 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T23:04:43.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:43 smithi181 ceph-mon[26930]: mgrmap e26: smithi181.whsrmz(active, since 9s), standbys: smithi144.rlueky 2024-04-12T23:04:44.231 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:44 smithi144 ceph-mon[24815]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.3 MiB/s wr, 405 op/s 2024-04-12T23:04:44.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 77232 9.66 MB/sec execute 5 sec latency 147.405 ms 2024-04-12T23:04:44.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:44 smithi181 ceph-mon[26930]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.3 MiB/s wr, 405 op/s 2024-04-12T23:04:45.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 77879 12.37 MB/sec execute 6 sec latency 24.879 ms 2024-04-12T23:04:46.345 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:46 smithi144 ceph-mon[24815]: pgmap v9: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.3 MiB/s wr, 642 op/s 2024-04-12T23:04:46.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 78464 13.74 MB/sec execute 7 sec latency 24.458 ms 2024-04-12T23:04:46.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:46 smithi181 ceph-mon[26930]: pgmap v9: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.3 MiB/s wr, 642 op/s 2024-04-12T23:04:47.069 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:04:47.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 78915 12.83 MB/sec execute 8 sec latency 41.149 ms 2024-04-12T23:04:48.153 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:04:48.153 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (7m) 2m ago 8m 22.4M - ba2b418f427c 5fe9fb07ff47 2024-04-12T23:04:48.153 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (7m) 0s ago 7m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (6m) 2m ago 8m 54.7M - 8.3.5 dad864ee21e9 79755181f90b 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (2m) 2m ago 2m 14.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (2m) 0s ago 2m 12.8M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (9m) 2m ago 9m 459M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (25s) 0s ago 7m 488M - 19.0.0-2053-gb277feb2 668895944f0e 54a7c01612f6 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (9m) 2m ago 9m 47.9M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (7m) 0s ago 7m 40.4M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (2m) 2m ago 2m 47.0M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (2m) 0s ago 2m 77.6M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 2m ago 8m 17.6M - 1dbe0e931976 7410a549b2f7 2024-04-12T23:04:48.154 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 0s ago 7m 20.5M - 1.3.1 1dbe0e931976 0d304d76f102 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (7m) 2m ago 6m 43.7M 3216M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (6m) 2m ago 6m 44.4M 3216M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (6m) 2m ago 6m 40.4M 3216M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (5m) 2m ago 5m 39.4M 3216M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (5m) 0s ago 5m 270M 3984M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (4m) 0s ago 4m 316M 3984M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (4m) 0s ago 4m 272M 3984M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (4m) 0s ago 4m 362M 3984M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:04:48.155 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (7m) 2m ago 7m 61.9M - 514e6a882f6e 3c5300d666ec 2024-04-12T23:04:48.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:48 smithi144 ceph-mon[24815]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:48.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:48 smithi144 ceph-mon[24815]: pgmap v10: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-04-12T23:04:48.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:48 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:48.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:48 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:48.360 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:48 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:04:48.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 79385 12.23 MB/sec execute 9 sec latency 37.665 ms 2024-04-12T23:04:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:48 smithi181 ceph-mon[26930]: from='client.14700 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:48 smithi181 ceph-mon[26930]: pgmap v10: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-04-12T23:04:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:48 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:48 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:48 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:04:48.806 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:04:48.807 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:04:48.808 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:04:49.378 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "1/23 daemons upgraded", 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "", 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:04:49.379 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:04:49.395 INFO:teuthology.orchestra.run.smithi144.stdout: 5 79846 12.84 MB/sec execute 10 sec latency 40.518 ms 2024-04-12T23:04:49.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:49.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:49.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:49 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2335679627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:49.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:49 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2335679627' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:04:50.169 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:04:50.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 80519 12.35 MB/sec execute 11 sec latency 34.879 ms 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 6.9 MiB/s wr, 696 op/s 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='client.24457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/2963067457' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.620 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.621 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.621 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.621 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.621 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.621 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.621 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:50 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:50.644 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 6.9 MiB/s wr, 696 op/s 2024-04-12T23:04:50.644 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='client.24457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:04:50.644 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/2963067457' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:04:50.644 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:50.644 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:50.644 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:50.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:50 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:51.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 80873 12.25 MB/sec execute 12 sec latency 30.292 ms 2024-04-12T23:04:51.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:51 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi144 to 1936M 2024-04-12T23:04:51.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:51 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi144 to 1936M 2024-04-12T23:04:52.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 81508 13.18 MB/sec execute 13 sec latency 31.738 ms 2024-04-12T23:04:52.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: pgmap v12: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 9.0 MiB/s wr, 876 op/s 2024-04-12T23:04:52.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:52.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:04:52.685 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:52 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: pgmap v12: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 9.0 MiB/s wr, 876 op/s 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-12T23:04:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:04:52.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:52 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:04:53.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 82027 13.78 MB/sec execute 14 sec latency 27.970 ms 2024-04-12T23:04:53.691 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:53 smithi144 ceph-mon[24815]: Adjusting osd_memory_target on smithi181 to 2704M 2024-04-12T23:04:53.691 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:53 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T23:04:53.691 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:53 smithi144 ceph-mon[24815]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T23:04:53.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:53 smithi181 ceph-mon[26930]: Adjusting osd_memory_target on smithi181 to 2704M 2024-04-12T23:04:53.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:53 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T23:04:53.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:53 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T23:04:54.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 82381 13.19 MB/sec execute 15 sec latency 37.842 ms 2024-04-12T23:04:54.699 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: pgmap v13: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 600 op/s 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.700 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:54 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:04:54.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:04:54.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:04:54.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: pgmap v13: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 600 op/s 2024-04-12T23:04:54.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:04:54.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:54.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:54 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:04:55.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 82850 12.66 MB/sec execute 16 sec latency 44.551 ms 2024-04-12T23:04:55.730 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:55 smithi144 ceph-mon[24815]: Reconfiguring alertmanager.smithi144 (dependencies changed)... 2024-04-12T23:04:55.730 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:55 smithi144 ceph-mon[24815]: Reconfiguring daemon alertmanager.smithi144 on smithi144 2024-04-12T23:04:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:55 smithi181 ceph-mon[26930]: Reconfiguring alertmanager.smithi144 (dependencies changed)... 2024-04-12T23:04:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:55 smithi181 ceph-mon[26930]: Reconfiguring daemon alertmanager.smithi144 on smithi144 2024-04-12T23:04:56.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 83303 12.99 MB/sec execute 17 sec latency 47.676 ms 2024-04-12T23:04:56.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:56 smithi144 ceph-mon[24815]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 901 op/s 2024-04-12T23:04:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:56 smithi181 ceph-mon[26930]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 901 op/s 2024-04-12T23:04:57.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 84055 12.89 MB/sec execute 18 sec latency 29.434 ms 2024-04-12T23:04:58.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 84610 13.20 MB/sec execute 19 sec latency 28.797 ms 2024-04-12T23:04:58.756 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:58 smithi144 ceph-mon[24815]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 616 op/s 2024-04-12T23:04:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:58 smithi181 ceph-mon[26930]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 616 op/s 2024-04-12T23:04:59.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 85301 13.82 MB/sec execute 20 sec latency 25.760 ms 2024-04-12T23:05:00.016 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:04:59 smithi144 ceph-mon[24815]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 786 op/s 2024-04-12T23:05:00.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:04:59 smithi181 ceph-mon[26930]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 786 op/s 2024-04-12T23:05:00.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 85701 13.79 MB/sec execute 21 sec latency 39.249 ms 2024-04-12T23:05:01.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 86221 13.43 MB/sec execute 22 sec latency 34.141 ms 2024-04-12T23:05:02.211 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:02 smithi144 ceph-mon[24815]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 9.1 MiB/s wr, 841 op/s 2024-04-12T23:05:02.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 86636 13.23 MB/sec execute 23 sec latency 38.221 ms 2024-04-12T23:05:02.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:02 smithi181 ceph-mon[26930]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 9.1 MiB/s wr, 841 op/s 2024-04-12T23:05:03.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 87424 13.53 MB/sec execute 24 sec latency 22.287 ms 2024-04-12T23:05:04.299 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:04 smithi144 ceph-mon[24815]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-04-12T23:05:04.299 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:04 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:04.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 87905 13.47 MB/sec execute 25 sec latency 34.534 ms 2024-04-12T23:05:04.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:04 smithi181 ceph-mon[26930]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-04-12T23:05:04.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:04 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:05.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 88518 13.89 MB/sec execute 26 sec latency 24.773 ms 2024-04-12T23:05:06.324 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:06 smithi144 ceph-mon[24815]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.7 MiB/s wr, 961 op/s 2024-04-12T23:05:06.396 INFO:teuthology.orchestra.run.smithi144.stdout: 5 89047 14.14 MB/sec execute 27 sec latency 62.125 ms 2024-04-12T23:05:06.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:06 smithi181 ceph-mon[26930]: pgmap v19: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.7 MiB/s wr, 961 op/s 2024-04-12T23:05:07.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 89372 13.88 MB/sec execute 28 sec latency 51.046 ms 2024-04-12T23:05:08.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:08 smithi144 ceph-mon[24815]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 661 op/s 2024-04-12T23:05:08.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 89817 13.51 MB/sec execute 29 sec latency 40.205 ms 2024-04-12T23:05:08.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:08 smithi181 ceph-mon[26930]: pgmap v20: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 661 op/s 2024-04-12T23:05:09.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 90142 13.29 MB/sec execute 30 sec latency 54.892 ms 2024-04-12T23:05:09.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:09 smithi181 ceph-mon[26930]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.4 MiB/s wr, 771 op/s 2024-04-12T23:05:10.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:09 smithi144 ceph-mon[24815]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 7.4 MiB/s wr, 771 op/s 2024-04-12T23:05:10.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 90753 13.50 MB/sec execute 31 sec latency 51.025 ms 2024-04-12T23:05:11.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 91361 13.43 MB/sec execute 32 sec latency 36.481 ms 2024-04-12T23:05:12.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 91790 13.55 MB/sec execute 33 sec latency 36.255 ms 2024-04-12T23:05:12.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:12 smithi144 ceph-mon[24815]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 758 op/s 2024-04-12T23:05:12.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:12 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:12 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:12 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.551 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:12 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.552 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:12 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T23:05:12.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:12 smithi181 ceph-mon[26930]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 758 op/s 2024-04-12T23:05:12.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:12 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:12 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:12 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:12 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:12.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:12 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T23:05:13.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 92398 13.75 MB/sec execute 34 sec latency 38.972 ms 2024-04-12T23:05:13.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:13 smithi144 ceph-mon[24815]: Reconfiguring grafana.smithi144 (dependencies changed)... 2024-04-12T23:05:13.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:13 smithi144 ceph-mon[24815]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T23:05:13.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:13 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T23:05:13.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:13 smithi144 ceph-mon[24815]: Reconfiguring daemon grafana.smithi144 on smithi144 2024-04-12T23:05:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:13 smithi181 ceph-mon[26930]: Reconfiguring grafana.smithi144 (dependencies changed)... 2024-04-12T23:05:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:13 smithi181 ceph-mon[26930]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-12T23:05:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:13 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-12T23:05:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:13 smithi181 ceph-mon[26930]: Reconfiguring daemon grafana.smithi144 on smithi144 2024-04-12T23:05:14.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 92832 13.82 MB/sec execute 35 sec latency 24.831 ms 2024-04-12T23:05:14.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:14 smithi144 ceph-mon[24815]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 594 op/s 2024-04-12T23:05:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:14 smithi181 ceph-mon[26930]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 594 op/s 2024-04-12T23:05:15.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 93269 13.51 MB/sec execute 36 sec latency 35.145 ms 2024-04-12T23:05:16.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 93577 13.30 MB/sec execute 37 sec latency 43.693 ms 2024-04-12T23:05:16.605 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:16 smithi144 ceph-mon[24815]: pgmap v24: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 904 op/s 2024-04-12T23:05:16.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:16 smithi181 ceph-mon[26930]: pgmap v24: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 904 op/s 2024-04-12T23:05:17.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 93989 13.32 MB/sec execute 38 sec latency 35.388 ms 2024-04-12T23:05:18.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 94580 13.27 MB/sec execute 39 sec latency 40.923 ms 2024-04-12T23:05:18.642 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:18 smithi144 ceph-mon[24815]: pgmap v25: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 576 op/s 2024-04-12T23:05:18.642 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:18 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:18.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:18 smithi181 ceph-mon[26930]: pgmap v25: 97 pgs: 97 active+clean; 336 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 576 op/s 2024-04-12T23:05:18.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:18 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:19.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 94963 13.18 MB/sec execute 40 sec latency 55.623 ms 2024-04-12T23:05:19.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:19 smithi144 ceph-mon[24815]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 620 op/s 2024-04-12T23:05:20.085 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:19 smithi181 ceph-mon[26930]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 620 op/s 2024-04-12T23:05:20.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 95274 13.14 MB/sec execute 41 sec latency 40.687 ms 2024-04-12T23:05:20.688 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:05:21.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 95739 13.25 MB/sec execute 42 sec latency 40.146 ms 2024-04-12T23:05:21.726 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 starting - - - - 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (8m) 33s ago 8m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (8m) 34s ago 8m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (7m) 33s ago 8m 55.1M - 8.3.5 dad864ee21e9 79755181f90b 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (3m) 33s ago 3m 37.2M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (3m) 34s ago 3m 12.8M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (10m) 33s ago 10m 379M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (58s) 34s ago 8m 488M - 19.0.0-2053-gb277feb2 668895944f0e 54a7c01612f6 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (10m) 33s ago 10m 51.5M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (8m) 34s ago 8m 40.4M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (2m) 33s ago 2m 243M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (2m) 34s ago 2m 77.6M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:05:21.727 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 33s ago 8m 18.9M - 1.3.1 1dbe0e931976 7410a549b2f7 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 34s ago 8m 20.5M - 1.3.1 1dbe0e931976 0d304d76f102 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (7m) 33s ago 7m 304M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (7m) 33s ago 7m 240M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (6m) 33s ago 6m 216M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (6m) 33s ago 6m 262M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (5m) 34s ago 5m 270M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (5m) 34s ago 5m 316M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (5m) 34s ago 5m 272M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (4m) 34s ago 4m 362M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:05:21.728 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (7m) 33s ago 7m 64.2M - 2.33.4 514e6a882f6e 3c5300d666ec 2024-04-12T23:05:22.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:22 smithi144 ceph-mon[24815]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:22.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:22 smithi144 ceph-mon[24815]: pgmap v27: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 729 op/s 2024-04-12T23:05:22.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:22 smithi144 ceph-mon[24815]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:05:22.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:05:22.381 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:05:22.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 96123 13.31 MB/sec execute 43 sec latency 31.472 ms 2024-04-12T23:05:22.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:22 smithi181 ceph-mon[26930]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:22 smithi181 ceph-mon[26930]: pgmap v27: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 729 op/s 2024-04-12T23:05:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:22 smithi181 ceph-mon[26930]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "1/23 daemons upgraded", 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "", 2024-04-12T23:05:22.912 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:05:22.913 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:05:23.236 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:23 smithi144 ceph-mon[24815]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:23.236 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:23 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1439494363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:05:23.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 96381 13.17 MB/sec execute 44 sec latency 42.815 ms 2024-04-12T23:05:23.408 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:05:23.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:23 smithi181 ceph-mon[26930]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:23.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:23 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1439494363' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:05:24.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:24 smithi144 ceph-mon[24815]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:24.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:24 smithi144 ceph-mon[24815]: pgmap v28: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 572 op/s 2024-04-12T23:05:24.247 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:24 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1240007914' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:05:24.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 96839 12.94 MB/sec execute 45 sec latency 38.839 ms 2024-04-12T23:05:24.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:24 smithi181 ceph-mon[26930]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:24.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:24 smithi181 ceph-mon[26930]: pgmap v28: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 572 op/s 2024-04-12T23:05:24.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:24 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1240007914' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:05:25.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 97227 12.88 MB/sec execute 46 sec latency 25.642 ms 2024-04-12T23:05:26.280 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:26 smithi144 ceph-mon[24815]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 785 op/s 2024-04-12T23:05:26.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 97806 12.95 MB/sec execute 47 sec latency 29.019 ms 2024-04-12T23:05:26.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:26 smithi181 ceph-mon[26930]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 785 op/s 2024-04-12T23:05:27.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 98374 12.89 MB/sec execute 48 sec latency 41.199 ms 2024-04-12T23:05:28.305 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:28 smithi144 ceph-mon[24815]: pgmap v30: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-04-12T23:05:28.397 INFO:teuthology.orchestra.run.smithi144.stdout: 5 98724 12.91 MB/sec execute 49 sec latency 55.104 ms 2024-04-12T23:05:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:28 smithi181 ceph-mon[26930]: pgmap v30: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-04-12T23:05:29.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 99207 13.02 MB/sec execute 50 sec latency 32.214 ms 2024-04-12T23:05:29.820 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:29 smithi144 ceph-mon[24815]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 670 op/s 2024-04-12T23:05:30.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:29 smithi181 ceph-mon[26930]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 670 op/s 2024-04-12T23:05:30.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 99639 13.03 MB/sec execute 51 sec latency 33.222 ms 2024-04-12T23:05:31.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 99988 12.99 MB/sec execute 52 sec latency 32.706 ms 2024-04-12T23:05:32.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:32 smithi144 ceph-mon[24815]: pgmap v32: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.0 MiB/s wr, 720 op/s 2024-04-12T23:05:32.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 100331 12.79 MB/sec execute 53 sec latency 39.931 ms 2024-04-12T23:05:32.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:32 smithi181 ceph-mon[26930]: pgmap v32: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.0 MiB/s wr, 720 op/s 2024-04-12T23:05:33.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 100653 12.65 MB/sec execute 54 sec latency 48.747 ms 2024-04-12T23:05:34.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:34 smithi144 ceph-mon[24815]: pgmap v33: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 501 op/s 2024-04-12T23:05:34.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:34.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:34 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:34.251 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:34 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:34.251 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:34 smithi144 ceph-mon[24815]: Reconfiguring node-exporter.smithi144 (dependencies changed)... 2024-04-12T23:05:34.251 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:34 smithi144 ceph-mon[24815]: Reconfiguring daemon node-exporter.smithi144 on smithi144 2024-04-12T23:05:34.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 100974 12.65 MB/sec execute 55 sec latency 69.917 ms 2024-04-12T23:05:34.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:34 smithi181 ceph-mon[26930]: pgmap v33: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 501 op/s 2024-04-12T23:05:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:34 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:34 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:34 smithi181 ceph-mon[26930]: Reconfiguring node-exporter.smithi144 (dependencies changed)... 2024-04-12T23:05:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:34 smithi181 ceph-mon[26930]: Reconfiguring daemon node-exporter.smithi144 on smithi144 2024-04-12T23:05:35.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 101314 12.53 MB/sec execute 56 sec latency 152.582 ms 2024-04-12T23:05:36.378 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:36 smithi144 ceph-mon[24815]: pgmap v34: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-04-12T23:05:36.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 101813 12.51 MB/sec execute 57 sec latency 40.299 ms 2024-04-12T23:05:36.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:36 smithi181 ceph-mon[26930]: pgmap v34: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-04-12T23:05:37.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 102126 12.45 MB/sec execute 58 sec latency 61.957 ms 2024-04-12T23:05:38.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 102525 12.49 MB/sec execute 59 sec latency 37.922 ms 2024-04-12T23:05:38.412 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:38 smithi144 ceph-mon[24815]: pgmap v35: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-04-12T23:05:38.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:38 smithi181 ceph-mon[26930]: pgmap v35: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 523 op/s 2024-04-12T23:05:39.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 102975 12.56 MB/sec execute 60 sec latency 39.658 ms 2024-04-12T23:05:40.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 103287 12.55 MB/sec execute 61 sec latency 41.080 ms 2024-04-12T23:05:40.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:40 smithi144 ceph-mon[24815]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 699 op/s 2024-04-12T23:05:40.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:40 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:40.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:40 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:40.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:40 smithi144 ceph-mon[24815]: Reconfiguring prometheus.smithi144 (dependencies changed)... 2024-04-12T23:05:40.444 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:40 smithi144 ceph-mon[24815]: Reconfiguring daemon prometheus.smithi144 on smithi144 2024-04-12T23:05:40.639 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:40 smithi181 ceph-mon[26930]: pgmap v36: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 699 op/s 2024-04-12T23:05:40.639 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:40 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:40.639 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:40 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:05:40.640 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:40 smithi181 ceph-mon[26930]: Reconfiguring prometheus.smithi144 (dependencies changed)... 2024-04-12T23:05:40.640 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:40 smithi181 ceph-mon[26930]: Reconfiguring daemon prometheus.smithi144 on smithi144 2024-04-12T23:05:41.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 103589 12.44 MB/sec execute 62 sec latency 45.298 ms 2024-04-12T23:05:42.398 INFO:teuthology.orchestra.run.smithi144.stdout: 5 104057 12.32 MB/sec execute 63 sec latency 35.769 ms 2024-04-12T23:05:42.472 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:42 smithi144 ceph-mon[24815]: pgmap v37: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 552 op/s 2024-04-12T23:05:42.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:42 smithi181 ceph-mon[26930]: pgmap v37: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 552 op/s 2024-04-12T23:05:43.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 104330 12.24 MB/sec execute 64 sec latency 46.847 ms 2024-04-12T23:05:44.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 104871 12.27 MB/sec execute 65 sec latency 32.864 ms 2024-04-12T23:05:44.515 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:44 smithi144 ceph-mon[24815]: pgmap v38: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 458 op/s 2024-04-12T23:05:44.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:44 smithi181 ceph-mon[26930]: pgmap v38: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 458 op/s 2024-04-12T23:05:45.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 105438 12.27 MB/sec execute 66 sec latency 31.846 ms 2024-04-12T23:05:46.297 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:46 smithi144 ceph-mon[24815]: pgmap v39: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 694 op/s 2024-04-12T23:05:46.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 105869 12.31 MB/sec execute 67 sec latency 25.283 ms 2024-04-12T23:05:46.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:46 smithi181 ceph-mon[26930]: pgmap v39: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 694 op/s 2024-04-12T23:05:47.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 106346 12.39 MB/sec execute 68 sec latency 32.918 ms 2024-04-12T23:05:48.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:48 smithi144 ceph-mon[24815]: pgmap v40: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 459 op/s 2024-04-12T23:05:48.351 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:48 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:48.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 106716 12.40 MB/sec execute 69 sec latency 41.111 ms 2024-04-12T23:05:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:48 smithi181 ceph-mon[26930]: pgmap v40: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 459 op/s 2024-04-12T23:05:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:48 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:05:49.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 106962 12.32 MB/sec execute 70 sec latency 43.826 ms 2024-04-12T23:05:49.878 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:49 smithi144 ceph-mon[24815]: pgmap v41: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 697 op/s 2024-04-12T23:05:50.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:49 smithi181 ceph-mon[26930]: pgmap v41: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.0 MiB/s wr, 697 op/s 2024-04-12T23:05:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 107434 12.24 MB/sec execute 71 sec latency 42.713 ms 2024-04-12T23:05:51.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 108020 12.27 MB/sec execute 72 sec latency 25.699 ms 2024-04-12T23:05:52.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 109049 12.45 MB/sec execute 73 sec latency 15.053 ms 2024-04-12T23:05:52.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:52 smithi144 ceph-mon[24815]: pgmap v42: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 540 op/s 2024-04-12T23:05:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:52 smithi181 ceph-mon[26930]: pgmap v42: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 540 op/s 2024-04-12T23:05:53.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 109872 12.67 MB/sec execute 74 sec latency 19.368 ms 2024-04-12T23:05:54.017 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:05:54.170 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:54 smithi144 ceph-mon[24815]: pgmap v43: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 492 op/s 2024-04-12T23:05:54.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 110322 12.71 MB/sec execute 75 sec latency 35.500 ms 2024-04-12T23:05:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:54 smithi181 ceph-mon[26930]: pgmap v43: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 492 op/s 2024-04-12T23:05:55.080 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 starting - - - - 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (9m) 66s ago 9m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (8m) 67s ago 8m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 starting - - - - 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (3m) 66s ago 3m 37.2M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (3m) 67s ago 3m 12.8M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:9283 running (10m) 66s ago 10m 379M - 17.2.0 e1d6a67b021e 4fac21237d56 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (91s) 67s ago 8m 488M - 19.0.0-2053-gb277feb2 668895944f0e 54a7c01612f6 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (10m) 66s ago 11m 51.5M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (8m) 67s ago 8m 40.4M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (3m) 66s ago 3m 243M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (3m) 67s ago 3m 77.6M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 starting - - - - 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 67s ago 8m 20.5M - 1.3.1 1dbe0e931976 0d304d76f102 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (8m) 66s ago 8m 304M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:05:55.081 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (7m) 66s ago 7m 240M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (7m) 66s ago 7m 216M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (6m) 66s ago 6m 262M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (6m) 67s ago 6m 270M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (6m) 67s ago 6m 316M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (5m) 67s ago 5m 272M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (5m) 67s ago 5m 362M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:05:55.082 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (8m) 66s ago 8m 64.2M - 2.33.4 514e6a882f6e 3c5300d666ec 2024-04-12T23:05:55.184 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:55 smithi144 ceph-mon[24815]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:55.184 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:55 smithi144 ceph-mon[24815]: from='client.24489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:55.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 110710 12.67 MB/sec execute 76 sec latency 28.722 ms 2024-04-12T23:05:55.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:55 smithi181 ceph-mon[26930]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:55.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:55 smithi181 ceph-mon[26930]: from='client.24489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:05:55.684 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:05:55.685 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:05:56.188 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:56 smithi144 ceph-mon[24815]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:56.188 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:56 smithi144 ceph-mon[24815]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 874 op/s 2024-04-12T23:05:56.188 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:56 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/378358495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "1/23 daemons upgraded", 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "", 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:05:56.199 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:05:56.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 111149 12.56 MB/sec execute 77 sec latency 39.883 ms 2024-04-12T23:05:56.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:56 smithi181 ceph-mon[26930]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:56.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:56 smithi181 ceph-mon[26930]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 874 op/s 2024-04-12T23:05:56.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:56 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/378358495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:05:56.899 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:05:57.202 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:57 smithi144 ceph-mon[24815]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:57.203 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:57 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1205681173' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:05:57.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 111472 12.50 MB/sec execute 78 sec latency 47.994 ms 2024-04-12T23:05:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:57 smithi181 ceph-mon[26930]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:05:57.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:57 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1205681173' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:05:58.391 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:58 smithi144 ceph-mon[24815]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-04-12T23:05:58.399 INFO:teuthology.orchestra.run.smithi144.stdout: 5 111901 12.51 MB/sec execute 79 sec latency 70.831 ms 2024-04-12T23:05:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:58 smithi181 ceph-mon[26930]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-04-12T23:05:59.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 112345 12.47 MB/sec execute 80 sec latency 45.367 ms 2024-04-12T23:05:59.920 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:05:59 smithi144 ceph-mon[24815]: pgmap v46: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 873 op/s 2024-04-12T23:06:00.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:05:59 smithi181 ceph-mon[26930]: pgmap v46: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.8 MiB/s wr, 873 op/s 2024-04-12T23:06:00.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 112781 12.47 MB/sec execute 81 sec latency 35.853 ms 2024-04-12T23:06:01.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 113256 12.53 MB/sec execute 82 sec latency 32.939 ms 2024-04-12T23:06:02.203 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:02 smithi144 ceph-mon[24815]: pgmap v47: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 652 op/s 2024-04-12T23:06:02.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 113715 12.56 MB/sec execute 83 sec latency 39.253 ms 2024-04-12T23:06:02.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:02 smithi181 ceph-mon[26930]: pgmap v47: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 652 op/s 2024-04-12T23:06:03.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 114067 12.55 MB/sec execute 84 sec latency 40.933 ms 2024-04-12T23:06:04.231 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:04 smithi144 ceph-mon[24815]: pgmap v48: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.2 MiB/s wr, 634 op/s 2024-04-12T23:06:04.231 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:04 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:04.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 114442 12.47 MB/sec execute 85 sec latency 39.185 ms 2024-04-12T23:06:04.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:04 smithi181 ceph-mon[26930]: pgmap v48: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.2 MiB/s wr, 634 op/s 2024-04-12T23:06:04.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:04 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:05.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 114852 12.42 MB/sec execute 86 sec latency 31.533 ms 2024-04-12T23:06:06.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:06 smithi144 ceph-mon[24815]: pgmap v49: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 882 op/s 2024-04-12T23:06:06.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:06 smithi144 ceph-mon[24815]: mgrmap e27: smithi181.whsrmz(active, since 92s), standbys: smithi144.rlueky 2024-04-12T23:06:06.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:06 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:06.248 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:06 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:06.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 115380 12.42 MB/sec execute 87 sec latency 54.912 ms 2024-04-12T23:06:06.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:06 smithi181 ceph-mon[26930]: pgmap v49: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.0 MiB/s wr, 882 op/s 2024-04-12T23:06:06.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:06 smithi181 ceph-mon[26930]: mgrmap e27: smithi181.whsrmz(active, since 92s), standbys: smithi144.rlueky 2024-04-12T23:06:06.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:06 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:06.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:06 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:07.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:07 smithi144 ceph-mon[24815]: Reconfiguring node-exporter.smithi181 (dependencies changed)... 2024-04-12T23:06:07.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:07 smithi144 ceph-mon[24815]: Reconfiguring daemon node-exporter.smithi181 on smithi181 2024-04-12T23:06:07.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 115888 12.42 MB/sec execute 88 sec latency 33.401 ms 2024-04-12T23:06:07.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:07 smithi181 ceph-mon[26930]: Reconfiguring node-exporter.smithi181 (dependencies changed)... 2024-04-12T23:06:07.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:07 smithi181 ceph-mon[26930]: Reconfiguring daemon node-exporter.smithi181 on smithi181 2024-04-12T23:06:08.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:08 smithi144 ceph-mon[24815]: pgmap v50: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 500 op/s 2024-04-12T23:06:08.321 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:08 smithi181 ceph-mon[26930]: pgmap v50: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 500 op/s 2024-04-12T23:06:08.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 116522 12.50 MB/sec execute 89 sec latency 44.995 ms 2024-04-12T23:06:09.400 INFO:teuthology.orchestra.run.smithi144.stdout: 5 117153 12.62 MB/sec execute 90 sec latency 22.911 ms 2024-04-12T23:06:09.533 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi144.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi144.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.534 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:09.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:09 smithi144 ceph-mon[24815]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi144.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi144.front.sepia.ceph.com:9093"}]: dispatch 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi144.front.sepia.ceph.com:3000"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi144.front.sepia.ceph.com:9095"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:09.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:09 smithi181 ceph-mon[26930]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:10.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 117659 12.64 MB/sec execute 91 sec latency 71.838 ms 2024-04-12T23:06:10.539 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:10 smithi144 ceph-mon[24815]: pgmap v51: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.8 MiB/s wr, 791 op/s 2024-04-12T23:06:10.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:10 smithi181 ceph-mon[26930]: pgmap v51: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.8 MiB/s wr, 791 op/s 2024-04-12T23:06:11.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 118149 12.58 MB/sec execute 92 sec latency 31.361 ms 2024-04-12T23:06:12.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 118552 12.53 MB/sec execute 93 sec latency 37.353 ms 2024-04-12T23:06:12.555 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: pgmap v52: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 578 op/s 2024-04-12T23:06:12.555 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: Upgrade: Updating mgr.smithi144.rlueky 2024-04-12T23:06:12.555 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:12.555 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:06:12.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:06:12.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:06:12.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:06:12.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:12 smithi144 ceph-mon[24815]: Deploying daemon mgr.smithi144.rlueky on smithi144 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: pgmap v52: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 578 op/s 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: Upgrade: Updating mgr.smithi144.rlueky 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:06:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:12 smithi181 ceph-mon[26930]: Deploying daemon mgr.smithi144.rlueky on smithi144 2024-04-12T23:06:13.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 119050 12.55 MB/sec execute 94 sec latency 37.110 ms 2024-04-12T23:06:14.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 119488 12.51 MB/sec execute 95 sec latency 55.778 ms 2024-04-12T23:06:14.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:14 smithi144 ceph-mon[24815]: pgmap v53: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.0 MiB/s wr, 562 op/s 2024-04-12T23:06:14.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:14 smithi181 ceph-mon[26930]: pgmap v53: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.0 MiB/s wr, 562 op/s 2024-04-12T23:06:15.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 120044 12.58 MB/sec execute 96 sec latency 57.072 ms 2024-04-12T23:06:16.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 120565 12.66 MB/sec execute 97 sec latency 43.807 ms 2024-04-12T23:06:16.606 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:16 smithi144 ceph-mon[24815]: pgmap v54: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-04-12T23:06:16.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:16 smithi181 ceph-mon[26930]: pgmap v54: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 859 op/s 2024-04-12T23:06:17.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 120983 12.65 MB/sec execute 98 sec latency 60.206 ms 2024-04-12T23:06:18.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 121405 12.63 MB/sec execute 99 sec latency 25.257 ms 2024-04-12T23:06:18.646 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:18 smithi144 ceph-mon[24815]: pgmap v55: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-04-12T23:06:18.646 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:18 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:18.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:18 smithi181 ceph-mon[26930]: pgmap v55: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-04-12T23:06:18.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:18 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:19.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 121826 12.55 MB/sec execute 100 sec latency 84.264 ms 2024-04-12T23:06:20.190 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:20 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.190 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:20 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.190 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:20 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:20.191 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:20 smithi144 ceph-mon[24815]: pgmap v56: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 871 op/s 2024-04-12T23:06:20.191 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:20 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.191 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:20 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:20 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:20 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:20 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:20.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:20 smithi181 ceph-mon[26930]: pgmap v56: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 871 op/s 2024-04-12T23:06:20.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:20 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.264 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:20 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:20.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 122165 12.52 MB/sec execute 101 sec latency 40.403 ms 2024-04-12T23:06:21.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 122610 12.51 MB/sec execute 102 sec latency 76.895 ms 2024-04-12T23:06:22.228 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:22 smithi144 ceph-mon[24815]: pgmap v57: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 595 op/s 2024-04-12T23:06:22.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 123021 12.47 MB/sec execute 103 sec latency 56.545 ms 2024-04-12T23:06:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:22 smithi181 ceph-mon[26930]: pgmap v57: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 595 op/s 2024-04-12T23:06:23.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 123493 12.50 MB/sec execute 104 sec latency 51.688 ms 2024-04-12T23:06:24.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 123887 12.52 MB/sec execute 105 sec latency 44.969 ms 2024-04-12T23:06:24.508 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:24 smithi144 ceph-mon[24815]: pgmap v58: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.8 MiB/s wr, 572 op/s 2024-04-12T23:06:24.508 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:24 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:24.508 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:24 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:24.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:24 smithi181 ceph-mon[26930]: pgmap v58: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.8 MiB/s wr, 572 op/s 2024-04-12T23:06:24.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:24 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:24.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:24 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:25.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 124328 12.54 MB/sec execute 106 sec latency 42.012 ms 2024-04-12T23:06:26.316 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:26 smithi144 ceph-mon[24815]: pgmap v59: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 817 op/s 2024-04-12T23:06:26.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 124693 12.53 MB/sec execute 107 sec latency 50.297 ms 2024-04-12T23:06:26.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:26 smithi181 ceph-mon[26930]: pgmap v59: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 817 op/s 2024-04-12T23:06:27.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 124995 12.46 MB/sec execute 108 sec latency 41.133 ms 2024-04-12T23:06:27.509 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:06:28.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:28 smithi144 ceph-mon[24815]: pgmap v60: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-04-12T23:06:28.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:28 smithi144 ceph-mon[24815]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:28.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:28 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:28.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:28 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:28.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 125397 12.39 MB/sec execute 109 sec latency 50.735 ms 2024-04-12T23:06:28.615 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:06:28.615 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (78s) 0s ago 10m 17.5M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (10m) 0s ago 10m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (9m) 5s ago 9m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 unknown 0s ago 9m - - 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (4m) 0s ago 4m 37.2M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (4m) 5s ago 4m 13.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (10s) 0s ago 11m 91.8M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (2m) 5s ago 9m 507M - 19.0.0-2053-gb277feb2 668895944f0e 54a7c01612f6 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (11m) 0s ago 11m 54.2M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (9m) 5s ago 9m 42.0M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (4m) 0s ago 4m 249M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (3m) 5s ago 3m 85.5M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (50s) 0s ago 9m 11.5M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (20s) 5s ago 9m 10.4M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (8m) 0s ago 8m 445M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (8m) 0s ago 8m 394M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:06:28.616 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (7m) 0s ago 7m 304M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:06:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (7m) 0s ago 7m 413M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:06:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (6m) 5s ago 6m 407M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:06:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (6m) 5s ago 6m 462M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:06:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (6m) 5s ago 6m 410M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:06:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (5m) 5s ago 5m 501M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:06:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (23s) 0s ago 8m 50.2M - 2.33.4 514e6a882f6e b155e6d8e5cd 2024-04-12T23:06:28.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:28 smithi181 ceph-mon[26930]: pgmap v60: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-04-12T23:06:28.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:28 smithi181 ceph-mon[26930]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:28.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:28 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:28.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:28 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:29.326 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:06:29.326 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:06:29.327 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:06:29.342 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: Standby manager daemon smithi144.rlueky restarted 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: Standby manager daemon smithi144.rlueky started 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/crt"}]: dispatch 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/key"}]: dispatch 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:06:29.343 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:29 smithi144 ceph-mon[24815]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:29.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 125934 12.42 MB/sec execute 110 sec latency 39.739 ms 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: Standby manager daemon smithi144.rlueky restarted 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: Standby manager daemon smithi144.rlueky started 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/crt"}]: dispatch 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi144.rlueky/key"}]: dispatch 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:06:29.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:29 smithi181 ceph-mon[26930]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:29.888 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:06:29.888 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr" 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "2/23 daemons upgraded", 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading mgr daemons", 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:06:29.889 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:06:30.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:30 smithi144 ceph-mon[24815]: pgmap v61: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 759 op/s 2024-04-12T23:06:30.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:30 smithi144 ceph-mon[24815]: mgrmap e28: smithi181.whsrmz(active, since 116s), standbys: smithi144.rlueky 2024-04-12T23:06:30.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:30 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/484461239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:06:30.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 126478 12.42 MB/sec execute 111 sec latency 41.907 ms 2024-04-12T23:06:30.424 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:06:30.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:30 smithi181 ceph-mon[26930]: pgmap v61: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 759 op/s 2024-04-12T23:06:30.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:30 smithi181 ceph-mon[26930]: mgrmap e28: smithi181.whsrmz(active, since 116s), standbys: smithi144.rlueky 2024-04-12T23:06:30.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:30 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/484461239' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:06:31.374 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:31 smithi144 ceph-mon[24815]: from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:31.374 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:31 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1311726271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:06:31.374 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:31 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:31.374 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:31 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:31.406 INFO:teuthology.orchestra.run.smithi144.stdout: 5 126882 12.41 MB/sec execute 112 sec latency 33.939 ms 2024-04-12T23:06:31.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:31 smithi181 ceph-mon[26930]: from='client.14792 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:06:31.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:31 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1311726271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:06:31.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:31 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:31.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:31 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:32.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 127201 12.39 MB/sec execute 113 sec latency 59.642 ms 2024-04-12T23:06:32.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:32 smithi144 ceph-mon[24815]: pgmap v62: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 512 op/s 2024-04-12T23:06:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:32 smithi181 ceph-mon[26930]: pgmap v62: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 512 op/s 2024-04-12T23:06:33.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 127609 12.42 MB/sec execute 114 sec latency 41.638 ms 2024-04-12T23:06:33.681 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:33 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:33.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:33 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:34.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 127907 12.39 MB/sec execute 115 sec latency 41.759 ms 2024-04-12T23:06:34.699 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:34 smithi144 ceph-mon[24815]: pgmap v63: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 497 op/s 2024-04-12T23:06:34.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:34 smithi181 ceph-mon[26930]: pgmap v63: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 497 op/s 2024-04-12T23:06:35.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 128179 12.38 MB/sec execute 116 sec latency 40.352 ms 2024-04-12T23:06:36.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 128448 12.31 MB/sec execute 117 sec latency 54.590 ms 2024-04-12T23:06:36.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: pgmap v64: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-04-12T23:06:36.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:06:36.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:06:36.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:36.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr fail", "who": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:06:36.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr fail", "who": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:06:36.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:36 smithi144 ceph-mon[24815]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: pgmap v64: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 733 op/s 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 172.21.15.181:0/27158642' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr fail", "who": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "mgr fail", "who": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:06:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:36 smithi181 ceph-mon[26930]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T23:06:37.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 128745 12.23 MB/sec execute 118 sec latency 44.992 ms 2024-04-12T23:06:37.747 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: pgmap v65: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 524 op/s 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: Failing over to other MGR 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd='[{"prefix": "mgr fail", "who": "smithi181.whsrmz"}]': finished 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: mgrmap e29: smithi144.rlueky(active, starting, since 0.86387s) 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:06:37.748 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: Manager daemon smithi144.rlueky is now available 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:06:37.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:37 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:06:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: pgmap v65: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.5 MiB/s wr, 524 op/s 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: Upgrade: Need to upgrade myself (mgr.smithi181.whsrmz) 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: Failing over to other MGR 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24411 ' entity='mgr.smithi181.whsrmz' cmd='[{"prefix": "mgr fail", "who": "smithi181.whsrmz"}]': finished 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: mgrmap e29: smithi144.rlueky(active, starting, since 0.86387s) 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:06:37.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: Manager daemon smithi144.rlueky is now available 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:06:37.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:37.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:06:37.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:37 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:06:38.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 128960 12.16 MB/sec execute 119 sec latency 98.578 ms 2024-04-12T23:06:39.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 129176 12.10 MB/sec execute 120 sec latency 74.284 ms 2024-04-12T23:06:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:39 smithi144 ceph-mon[24815]: [12/Apr/2024:23:06:37] ENGINE Bus STARTING 2024-04-12T23:06:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:39 smithi144 ceph-mon[24815]: [12/Apr/2024:23:06:38] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T23:06:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:39 smithi144 ceph-mon[24815]: [12/Apr/2024:23:06:38] ENGINE Serving on http://172.21.15.144:8765 2024-04-12T23:06:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:39 smithi144 ceph-mon[24815]: [12/Apr/2024:23:06:38] ENGINE Bus STARTED 2024-04-12T23:06:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:39 smithi144 ceph-mon[24815]: mgrmap e30: smithi144.rlueky(active, since 1.89413s) 2024-04-12T23:06:39.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:39 smithi144 ceph-mon[24815]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:39.646 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:39 smithi181 ceph-mon[26930]: [12/Apr/2024:23:06:37] ENGINE Bus STARTING 2024-04-12T23:06:39.646 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:39 smithi181 ceph-mon[26930]: [12/Apr/2024:23:06:38] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T23:06:39.646 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:39 smithi181 ceph-mon[26930]: [12/Apr/2024:23:06:38] ENGINE Serving on http://172.21.15.144:8765 2024-04-12T23:06:39.646 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:39 smithi181 ceph-mon[26930]: [12/Apr/2024:23:06:38] ENGINE Bus STARTED 2024-04-12T23:06:39.646 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:39 smithi181 ceph-mon[26930]: mgrmap e30: smithi144.rlueky(active, since 1.89413s) 2024-04-12T23:06:39.646 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:39 smithi181 ceph-mon[26930]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:40.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 129504 12.08 MB/sec execute 121 sec latency 60.077 ms 2024-04-12T23:06:40.606 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:40 smithi144 ceph-mon[24815]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T23:06:40.607 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:40 smithi144 ceph-mon[24815]: Cluster is now healthy 2024-04-12T23:06:40.607 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:40 smithi144 ceph-mon[24815]: pgmap v4: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:40.607 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:40 smithi144 ceph-mon[24815]: mgrmap e31: smithi144.rlueky(active, since 2s) 2024-04-12T23:06:40.651 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:40 smithi181 ceph-mon[26930]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T23:06:40.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:40 smithi181 ceph-mon[26930]: Cluster is now healthy 2024-04-12T23:06:40.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:40 smithi181 ceph-mon[26930]: pgmap v4: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:40.652 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:40 smithi181 ceph-mon[26930]: mgrmap e31: smithi144.rlueky(active, since 2s) 2024-04-12T23:06:41.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 130059 12.09 MB/sec execute 122 sec latency 38.931 ms 2024-04-12T23:06:42.407 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:42 smithi144 ceph-mon[24815]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:42.407 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:42 smithi144 ceph-mon[24815]: mgrmap e32: smithi144.rlueky(active, since 4s) 2024-04-12T23:06:42.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 130517 12.10 MB/sec execute 123 sec latency 74.618 ms 2024-04-12T23:06:42.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:42 smithi181 ceph-mon[26930]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:42.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:42 smithi181 ceph-mon[26930]: mgrmap e32: smithi144.rlueky(active, since 4s) 2024-04-12T23:06:43.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 130948 12.13 MB/sec execute 124 sec latency 43.050 ms 2024-04-12T23:06:43.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:43 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:43.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:43 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:43.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:43 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:43.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:43 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:44.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 131247 12.13 MB/sec execute 125 sec latency 58.304 ms 2024-04-12T23:06:44.941 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:44 smithi144 ceph-mon[24815]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:44.941 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:44 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:44.941 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:44 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:45.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:44 smithi181 ceph-mon[26930]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:06:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:44 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:44 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:45.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 131498 12.08 MB/sec execute 126 sec latency 60.795 ms 2024-04-12T23:06:46.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 131741 12.06 MB/sec execute 127 sec latency 54.898 ms 2024-04-12T23:06:46.983 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:46 smithi144 ceph-mon[24815]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.0 MiB/s wr, 418 op/s 2024-04-12T23:06:46.983 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:46 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:06:46.983 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:46 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:06:46.983 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:46 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:06:46.983 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:46 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:06:46.984 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:46 smithi144 ceph-mon[24815]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:06:47.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:46 smithi181 ceph-mon[26930]: pgmap v7: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 4.0 MiB/s wr, 418 op/s 2024-04-12T23:06:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:46 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:06:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:46 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:06:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:46 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:06:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:46 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1537837583' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:06:47.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:46 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:06:47.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 132023 12.00 MB/sec execute 128 sec latency 60.699 ms 2024-04-12T23:06:48.002 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:47 smithi144 ceph-mon[24815]: mgrmap e33: smithi144.rlueky(active, since 10s), standbys: smithi181.whsrmz 2024-04-12T23:06:48.002 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:47 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:06:48.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:47 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:48.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:47 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:48.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:47 smithi181 ceph-mon[26930]: mgrmap e33: smithi144.rlueky(active, since 10s), standbys: smithi181.whsrmz 2024-04-12T23:06:48.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:47 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:06:48.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:47 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:48.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:47 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:48.407 INFO:teuthology.orchestra.run.smithi144.stdout: 5 132285 11.93 MB/sec execute 129 sec latency 59.114 ms 2024-04-12T23:06:49.018 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:48 smithi144 ceph-mon[24815]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.1 MiB/s wr, 324 op/s 2024-04-12T23:06:49.018 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:48 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:49.018 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:48 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:49.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:48 smithi181 ceph-mon[26930]: pgmap v8: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 3.1 MiB/s wr, 324 op/s 2024-04-12T23:06:49.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:48 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:49.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:48 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:49.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 132618 11.89 MB/sec execute 130 sec latency 39.465 ms 2024-04-12T23:06:50.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 133005 11.88 MB/sec execute 131 sec latency 58.281 ms 2024-04-12T23:06:51.060 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:50 smithi144 ceph-mon[24815]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-04-12T23:06:51.148 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:50 smithi181 ceph-mon[26930]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.5 MiB/s wr, 451 op/s 2024-04-12T23:06:51.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 133587 11.90 MB/sec execute 132 sec latency 28.624 ms 2024-04-12T23:06:52.083 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:51 smithi144 ceph-mon[24815]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-04-12T23:06:52.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:51 smithi181 ceph-mon[26930]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-04-12T23:06:52.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 134110 11.93 MB/sec execute 133 sec latency 39.196 ms 2024-04-12T23:06:53.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 134472 11.94 MB/sec execute 134 sec latency 50.524 ms 2024-04-12T23:06:53.445 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:53 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:53.445 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:53 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:53 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:53.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:53 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:06:54.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 134784 11.93 MB/sec execute 135 sec latency 52.129 ms 2024-04-12T23:06:55.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:54 smithi144 ceph-mon[24815]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 437 op/s 2024-04-12T23:06:55.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:54 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:55.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:54 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:55.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:54 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:06:55.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:54 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:06:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:54 smithi181 ceph-mon[26930]: pgmap v11: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 437 op/s 2024-04-12T23:06:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:54 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:54 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:54 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:06:55.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:54 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:06:55.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 134995 11.88 MB/sec execute 136 sec latency 114.222 ms 2024-04-12T23:06:56.076 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T23:06:56.076 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T23:06:56.076 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:06:56.076 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:06:56.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.077 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:55 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:55 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:06:56.408 INFO:teuthology.orchestra.run.smithi144.stdout: 5 135218 11.87 MB/sec execute 137 sec latency 57.336 ms 2024-04-12T23:06:57.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:06:57.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:06:57.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:06:57.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:06:57.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.7 MiB/s wr, 671 op/s 2024-04-12T23:06:57.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-04-12T23:06:57.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: Reconfiguring prometheus.smithi144 (dependencies changed)... 2024-04-12T23:06:57.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:56 smithi144 ceph-mon[24815]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T23:06:57.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 6.7 MiB/s wr, 671 op/s 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: Reconfiguring prometheus.smithi144 (dependencies changed)... 2024-04-12T23:06:57.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:56 smithi181 ceph-mon[26930]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T23:06:57.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 135605 11.83 MB/sec execute 138 sec latency 56.586 ms 2024-04-12T23:06:58.091 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:57 smithi144 ceph-mon[24815]: Reconfiguring daemon prometheus.smithi144 on smithi144 2024-04-12T23:06:58.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:57 smithi181 ceph-mon[26930]: Reconfiguring daemon prometheus.smithi144 on smithi144 2024-04-12T23:06:58.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 136069 11.81 MB/sec execute 139 sec latency 43.348 ms 2024-04-12T23:06:58.844 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:06:58 smithi144 ceph-mon[24815]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-04-12T23:06:59.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:06:58 smithi181 ceph-mon[26930]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 503 op/s 2024-04-12T23:06:59.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 136463 11.77 MB/sec execute 140 sec latency 41.595 ms 2024-04-12T23:07:00.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 136962 11.78 MB/sec execute 141 sec latency 37.848 ms 2024-04-12T23:07:00.867 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:00 smithi144 ceph-mon[24815]: pgmap v15: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-04-12T23:07:01.016 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:07:01.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:00 smithi181 ceph-mon[26930]: pgmap v15: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-04-12T23:07:01.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 137356 11.79 MB/sec execute 142 sec latency 48.565 ms 2024-04-12T23:07:01.961 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:07:01.961 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (111s) 14s ago 10m 17.5M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:07:01.961 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (10m) 14s ago 10m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:07:01.961 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (10m) 19s ago 10m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:07:01.961 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 unknown 14s ago 10m - - 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (4m) 14s ago 4m 37.4M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (4m) 19s ago 4m 13.6M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (43s) 14s ago 12m 479M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283 running (2m) 19s ago 9m 74.6M - 19.0.0-2053-gb277feb2 668895944f0e 54a7c01612f6 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (12m) 14s ago 12m 52.6M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (9m) 19s ago 9m 41.3M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (4m) 14s ago 4m 261M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (4m) 19s ago 4m 85.7M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (84s) 14s ago 10m 8690k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (54s) 19s ago 9m 11.7M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:07:01.962 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (9m) 14s ago 9m 462M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (8m) 14s ago 8m 403M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (8m) 14s ago 8m 315M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (7m) 14s ago 7m 424M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (7m) 19s ago 7m 436M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (7m) 19s ago 7m 495M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (6m) 19s ago 6m 418M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (6m) 19s ago 6m 545M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:07:01.963 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (56s) 14s ago 9m 51.0M - 2.33.4 514e6a882f6e b155e6d8e5cd 2024-04-12T23:07:02.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 137749 11.78 MB/sec execute 143 sec latency 42.744 ms 2024-04-12T23:07:02.570 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:07:02.570 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:07:02.570 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:07:02.570 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:07:02.571 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:07:02.909 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:02 smithi144 ceph-mon[24815]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:02.909 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:02 smithi144 ceph-mon[24815]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:02.909 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:02 smithi144 ceph-mon[24815]: pgmap v16: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-04-12T23:07:02.909 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:02 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/1225431010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr" 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "2/23 daemons upgraded", 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "", 2024-04-12T23:07:03.094 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:07:03.095 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:07:03.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:02 smithi181 ceph-mon[26930]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:03.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:02 smithi181 ceph-mon[26930]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:03.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:02 smithi181 ceph-mon[26930]: pgmap v16: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-04-12T23:07:03.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:02 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/1225431010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:03.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 138071 11.78 MB/sec execute 144 sec latency 54.237 ms 2024-04-12T23:07:03.746 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:07:03.746 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:07:03.746 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:07:03.951 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:03 smithi144 ceph-mon[24815]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:03.951 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:03 smithi144 ceph-mon[24815]: from='client.? 172.21.15.144:0/169588428' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:07:03.951 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:03 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:04.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:03 smithi181 ceph-mon[26930]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:03 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/169588428' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:07:04.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:03 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:04.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 138423 11.79 MB/sec execute 145 sec latency 43.438 ms 2024-04-12T23:07:04.965 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:04 smithi144 ceph-mon[24815]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:04.965 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:04 smithi144 ceph-mon[24815]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 788 op/s 2024-04-12T23:07:04.965 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:04 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:04.965 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:04 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:07:04.965 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:04 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:04 smithi181 ceph-mon[26930]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:04 smithi181 ceph-mon[26930]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 788 op/s 2024-04-12T23:07:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:04 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:04 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:07:05.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:04 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:05.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 138801 11.79 MB/sec execute 146 sec latency 36.437 ms 2024-04-12T23:07:05.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:05 smithi144 ceph-mon[24815]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:07:05.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:05 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:07:05.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:05 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:07:06.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:05 smithi181 ceph-mon[26930]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:07:06.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:05 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:07:06.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:05 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:07:06.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 139352 11.77 MB/sec execute 147 sec latency 31.658 ms 2024-04-12T23:07:06.991 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:06 smithi144 ceph-mon[24815]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-04-12T23:07:06.991 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:06 smithi144 ceph-mon[24815]: Upgrade: Updating mgr.smithi181.whsrmz 2024-04-12T23:07:06.991 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:06 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:07:06.992 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:06 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:06.992 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:06 smithi144 ceph-mon[24815]: Deploying daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T23:07:07.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:06 smithi181 ceph-mon[26930]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-04-12T23:07:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:06 smithi181 ceph-mon[26930]: Upgrade: Updating mgr.smithi181.whsrmz 2024-04-12T23:07:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:06 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:07:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:06 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:07.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:06 smithi181 ceph-mon[26930]: Deploying daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T23:07:07.409 INFO:teuthology.orchestra.run.smithi144.stdout: 5 139865 11.77 MB/sec execute 148 sec latency 33.949 ms 2024-04-12T23:07:08.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 140539 11.80 MB/sec execute 149 sec latency 29.341 ms 2024-04-12T23:07:08.499 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:08 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:08.499 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:08 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:08.499 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:08 smithi144 ceph-mon[24815]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-04-12T23:07:08.636 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:08 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:08.636 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:08 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:08.636 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:08 smithi181 ceph-mon[26930]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-04-12T23:07:09.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 141112 11.83 MB/sec execute 150 sec latency 23.858 ms 2024-04-12T23:07:10.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 141676 11.88 MB/sec execute 151 sec latency 44.022 ms 2024-04-12T23:07:10.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:10 smithi181 ceph-mon[26930]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 735 op/s 2024-04-12T23:07:10.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:10 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:10.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:10 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:10.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:10 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:11.032 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:10 smithi144 ceph-mon[24815]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 735 op/s 2024-04-12T23:07:11.032 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:10 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:11.032 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:10 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:11.032 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:10 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:11.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 142165 11.92 MB/sec execute 152 sec latency 41.072 ms 2024-04-12T23:07:12.071 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:11 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:12.071 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:11 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:12.071 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:11 smithi144 ceph-mon[24815]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 534 op/s 2024-04-12T23:07:12.142 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:11 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:12.142 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:11 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:12.142 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:11 smithi181 ceph-mon[26930]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 534 op/s 2024-04-12T23:07:12.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 142572 11.91 MB/sec execute 153 sec latency 40.164 ms 2024-04-12T23:07:13.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 143041 11.86 MB/sec execute 154 sec latency 42.053 ms 2024-04-12T23:07:14.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 143526 11.87 MB/sec execute 155 sec latency 48.694 ms 2024-04-12T23:07:14.856 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:14 smithi144 ceph-mon[24815]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 838 op/s 2024-04-12T23:07:15.133 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:14 smithi181 ceph-mon[26930]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 838 op/s 2024-04-12T23:07:15.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 144091 11.87 MB/sec execute 156 sec latency 46.854 ms 2024-04-12T23:07:16.385 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:16 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:16.385 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:16 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:16.385 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:16 smithi144 ceph-mon[24815]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-04-12T23:07:16.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 144602 11.90 MB/sec execute 157 sec latency 48.345 ms 2024-04-12T23:07:16.524 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:16 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:16.524 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:16 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:16.525 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:16 smithi181 ceph-mon[26930]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-04-12T23:07:17.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 145142 11.94 MB/sec execute 158 sec latency 34.441 ms 2024-04-12T23:07:17.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:17 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:17.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:17 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:18.195 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:17 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:18.195 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:17 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:18.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 145562 11.96 MB/sec execute 159 sec latency 33.717 ms 2024-04-12T23:07:18.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:18 smithi144 ceph-mon[24815]: pgmap v24: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-04-12T23:07:18.913 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:18 smithi181 ceph-mon[26930]: pgmap v24: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-04-12T23:07:19.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 145988 11.96 MB/sec execute 160 sec latency 38.076 ms 2024-04-12T23:07:19.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:19 smithi144 ceph-mon[24815]: Standby manager daemon smithi181.whsrmz restarted 2024-04-12T23:07:19.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:19 smithi144 ceph-mon[24815]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:07:19.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:19 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:07:19.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:19 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:07:19.883 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:19 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:07:19.884 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:19 smithi144 ceph-mon[24815]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:07:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:19 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz restarted 2024-04-12T23:07:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:19 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:07:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:19 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:07:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:19 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:07:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:19 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:07:20.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:19 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/1058308268' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:07:20.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 146464 11.93 MB/sec execute 161 sec latency 29.133 ms 2024-04-12T23:07:20.898 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:20 smithi144 ceph-mon[24815]: pgmap v25: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 896 op/s 2024-04-12T23:07:20.898 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:20 smithi144 ceph-mon[24815]: mgrmap e34: smithi144.rlueky(active, since 43s), standbys: smithi181.whsrmz 2024-04-12T23:07:20.898 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:20 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:20.898 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:20 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:20.908 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:20 smithi181 ceph-mon[26930]: pgmap v25: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 896 op/s 2024-04-12T23:07:20.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:20 smithi181 ceph-mon[26930]: mgrmap e34: smithi144.rlueky(active, since 43s), standbys: smithi181.whsrmz 2024-04-12T23:07:20.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:20 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:20.909 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:20 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:21.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 146767 11.90 MB/sec execute 162 sec latency 60.346 ms 2024-04-12T23:07:22.181 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.181 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: pgmap v26: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.7 MiB/s wr, 632 op/s 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 766 op/s 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: Upgrade: Setting container_image for all mgr 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.rlueky"}]: dispatch 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.rlueky"}]: dispatch 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.rlueky"}]': finished 2024-04-12T23:07:22.182 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.whsrmz"}]: dispatch 2024-04-12T23:07:22.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.whsrmz"}]: dispatch 2024-04-12T23:07:22.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.whsrmz"}]': finished 2024-04-12T23:07:22.183 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:22 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T23:07:22.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 147084 11.87 MB/sec execute 163 sec latency 63.284 ms 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: pgmap v26: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.7 MiB/s wr, 632 op/s 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 766 op/s 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: Upgrade: Setting container_image for all mgr 2024-04-12T23:07:22.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.rlueky"}]: dispatch 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.rlueky"}]: dispatch 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi144.rlueky"}]': finished 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.whsrmz"}]: dispatch 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.whsrmz"}]: dispatch 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi181.whsrmz"}]': finished 2024-04-12T23:07:22.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:22 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T23:07:23.195 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:23 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:23.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 147440 11.86 MB/sec execute 164 sec latency 53.098 ms 2024-04-12T23:07:23.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:23 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:24.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 147959 11.88 MB/sec execute 165 sec latency 50.510 ms 2024-04-12T23:07:24.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:24 smithi144 ceph-mon[24815]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 378 op/s 2024-04-12T23:07:24.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:24 smithi144 ceph-mon[24815]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:24.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:24 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:07:24.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:24 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:07:24.978 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:24 smithi144 ceph-mon[24815]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:25.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:24 smithi181 ceph-mon[26930]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 378 op/s 2024-04-12T23:07:25.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:24 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:24 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:07:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:24 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:07:25.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:24 smithi181 ceph-mon[26930]: from='mgr.24501 172.21.15.144:0/2736643779' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:25.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 148475 11.89 MB/sec execute 166 sec latency 28.619 ms 2024-04-12T23:07:25.985 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:25 smithi144 ceph-mon[24815]: Upgrade: Updating mon.smithi144 2024-04-12T23:07:25.985 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:25 smithi144 ceph-mon[24815]: Deploying daemon mon.smithi144 on smithi144 2024-04-12T23:07:26.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:25 smithi181 ceph-mon[26930]: Upgrade: Updating mon.smithi144 2024-04-12T23:07:26.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:25 smithi181 ceph-mon[26930]: Deploying daemon mon.smithi144 on smithi144 2024-04-12T23:07:26.410 INFO:teuthology.orchestra.run.smithi144.stdout: 5 149004 11.93 MB/sec execute 167 sec latency 25.572 ms 2024-04-12T23:07:27.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:26 smithi144 ceph-mon[24815]: pgmap v29: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-12T23:07:27.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:26 smithi181 ceph-mon[26930]: pgmap v29: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-12T23:07:27.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:27 smithi144 systemd[1]: Stopping Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:07:27.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 149385 11.93 MB/sec execute 168 sec latency 81.324 ms 2024-04-12T23:07:27.512 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:27 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[24811]: 2024-04-12T23:07:27.399+0000 7fb94cfb5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi144 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:07:27.512 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:27 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[24811]: 2024-04-12T23:07:27.399+0000 7fb94cfb5700 -1 mon.smithi144@0(leader) e2 *** Got Signal Terminated *** 2024-04-12T23:07:27.512 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:27 smithi144 podman[66875]: 2024-04-12 23:07:27.459512658 +0000 UTC m=+0.206602637 container died cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, io.openshift.tags=base centos centos-stream, vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, release=754, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, ceph=True) 2024-04-12T23:07:28.279 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:28 smithi144 podman[66875]: 2024-04-12 23:07:28.258755955 +0000 UTC m=+1.005845928 container cleanup cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., version=8, architecture=x86_64, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, ceph=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T23:07:28.279 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:28 smithi144 bash[66875]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144 2024-04-12T23:07:28.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 149780 11.92 MB/sec execute 169 sec latency 34.829 ms 2024-04-12T23:07:29.043 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:28 smithi144 podman[66889]: 2024-04-12 23:07:28.81193124 +0000 UTC m=+1.353690548 container remove cdb5c41140d5f381f6ed4e62d3a064f36b51b16f5e70f278e02df32743194b91 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754) 2024-04-12T23:07:29.297 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:29 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service: Deactivated successfully. 2024-04-12T23:07:29.297 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:29 smithi144 systemd[1]: Stopped Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:07:29.297 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:29 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service: Consumed 16.398s CPU time. 2024-04-12T23:07:29.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 150131 11.87 MB/sec execute 170 sec latency 63.818 ms 2024-04-12T23:07:29.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:29 smithi144 systemd[1]: Starting Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:07:29.807 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:29 smithi144 podman[67013]: 2024-04-12 23:07:29.680834084 +0000 UTC m=+0.030479212 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:07:30.058 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:29 smithi144 podman[67013]: 2024-04-12 23:07:29.888596791 +0000 UTC m=+0.238241917 container create 70ffa37b271436f8a5f89a8139044746da10c894e0a23fd6e6f070ab5358a1a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-12T23:07:30.312 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 podman[67013]: 2024-04-12 23:07:30.28641245 +0000 UTC m=+0.636057591 container init 70ffa37b271436f8a5f89a8139044746da10c894e0a23fd6e6f070ab5358a1a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-04-12T23:07:30.312 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 podman[67013]: 2024-04-12 23:07:30.295716605 +0000 UTC m=+0.645361739 container start 70ffa37b271436f8a5f89a8139044746da10c894e0a23fd6e6f070ab5358a1a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:07:30.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 150530 11.87 MB/sec execute 171 sec latency 30.349 ms 2024-04-12T23:07:30.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T23:07:30.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T23:07:30.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: pidfile_write: ignore empty --pid-file 2024-04-12T23:07:30.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: load: jerasure load: lrc 2024-04-12T23:07:30.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: RocksDB version: 7.9.2 2024-04-12T23:07:30.561 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Git sha 0 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: DB SUMMARY 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: DB Session ID: 6WV5CZXDQCD4TZSCCKB9 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: CURRENT file: CURRENT 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi144/store.db dir, Total Num: 1, files: 000030.sst 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi144/store.db: 000028.log size: 2328386 ; 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.error_if_exists: 0 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.create_if_missing: 0 2024-04-12T23:07:30.562 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.paranoid_checks: 1 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.env: 0x5614ed08fc60 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.info_log: 0x5614ef88d9a0 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.statistics: (nil) 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.use_fsync: 0 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_log_file_size: 0 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T23:07:30.563 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.allow_fallocate: 1 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.use_direct_reads: 0 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.db_log_dir: 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.wal_dir: 2024-04-12T23:07:30.564 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T23:07:30.565 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T23:07:30.565 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.write_buffer_manager: 0x5614ef8601e0 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T23:07:30.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.unordered_write: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.row_cache: None 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.wal_filter: None 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.two_write_queues: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.wal_compression: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.atomic_flush: 0 2024-04-12T23:07:30.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.log_readahead_size: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_background_jobs: 2 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_background_compactions: -1 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_subcompactions: 1 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T23:07:30.568 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_open_files: -1 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_background_flushes: -1 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Compression algorithms supported: 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T23:07:30.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kZSTD supported: 0 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kXpressCompression supported: 0 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kZlibCompression supported: 1 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kSnappyCompression supported: 1 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kLZ4Compression supported: 1 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: kBZip2Compression supported: 0 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi144/store.db/MANIFEST-000015 2024-04-12T23:07:30.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.merge_operator: 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_filter: None 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5614ef88daa0) 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: cache_index_and_filter_blocks: 1 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: pin_top_level_index_and_filter: 1 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_type: 0 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_block_index_type: 0 2024-04-12T23:07:30.572 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_shortening: 1 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: checksum: 4 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: no_block_cache: 0 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache: 0x5614ef891090 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_name: BinnedLRUCache 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_options: 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: capacity : 536870912 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: num_shard_bits : 4 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: strict_capacity_limit : 0 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: high_pri_pool_ratio: 0.000 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_cache_compressed: (nil) 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: persistent_cache: (nil) 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_size: 4096 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_size_deviation: 10 2024-04-12T23:07:30.573 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_restart_interval: 16 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: index_block_restart_interval: 1 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: metadata_block_size: 4096 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: partition_filters: 0 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: use_delta_encoding: 1 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: filter_policy: bloomfilter 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: whole_key_filtering: 1 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: verify_compression: 0 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: read_amp_bytes_per_bit: 0 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: format_version: 5 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: enable_index_compression: 1 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: block_align: 0 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_auto_readahead_size: 262144 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: prepopulate_block_cache: 0 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: initial_auto_readahead_size: 8192 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T23:07:30.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression: NoCompression 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.num_levels: 7 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T23:07:30.575 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T23:07:30.576 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T23:07:30.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.table_properties_collectors: 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.inplace_update_support: 0 2024-04-12T23:07:30.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.bloom_locality: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.max_successive_merges: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.ttl: 2592000 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.enable_blob_files: false 2024-04-12T23:07:30.579 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.min_blob_size: 0 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi144/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11838, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 205fa14b-42bf-4c09-8673-95bf641a1ec9 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712963250337539, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-04-12T23:07:30.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712963250346055, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2127582, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11839, "largest_seqno": 12914, "table_properties": {"data_size": 2122227, "index_size": 3120, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10799, "raw_average_key_size": 23, "raw_value_size": 2112140, "raw_average_value_size": 4532, "num_data_blocks": 143, "num_entries": 466, "num_filter_entries": 466, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712963250, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "205fa14b-42bf-4c09-8673-95bf641a1ec9", "db_session_id": "6WV5CZXDQCD4TZSCCKB9", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712963250346210, "job": 1, "event": "recovery_finished"} 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi144/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5614ef982000 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: DB pointer 0x5614ef96c000 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** DB Stats ** 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T23:07:30.581 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: L0 1/0 2.03 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 508.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: L6 1/0 6.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Sum 2/0 8.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 508.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 508.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** Compaction Stats [default] ** 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 508.7 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-04-12T23:07:30.582 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Cumulative compaction: 0.00 GB write, 136.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Interval compaction: 0.00 GB write, 136.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Block cache BinnedLRUCache@0x5614ef891090#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T23:07:30.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: starting mon.smithi144 rank 0 at public addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] at bind addrs [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi144 fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???) e2 preinit fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).mds e7 new map 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).mds e7 print_map 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: e7 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: legacy client fscid: 1 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Filesystem 'foofs' (1) 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: fs_name foofs 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: epoch 5 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: created 2024-04-12T23:02:03.352809+0000 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: modified 2024-04-12T23:02:09.376571+0000 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: tableserver 0 2024-04-12T23:07:30.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: root 0 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: session_timeout 60 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: session_autoclose 300 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_file_size 1099511627776 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_xattr_size 65536 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: required_client_features {} 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: last_failure 0 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: last_failure_osd_epoch 0 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: max_mds 1 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: in 0 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: up {0=14490} 2024-04-12T23:07:30.585 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: failed 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: damaged 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: stopped 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: data_pools [3] 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: metadata_pool 2 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: inline_data disabled 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: balancer 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: bal_rank_mask -1 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: standby_count_wanted 1 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: qdb_cluster leader: 0 members: 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: [mds.foofs.smithi144.alilcy{0:14490} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.144:6834/778305732,v1:172.21.15.144:6835/778305732] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: Standby daemons: 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: 2024-04-12T23:07:30.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout: [mds.foofs.smithi181.gciyns{-1:24267} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 bash[67013]: 70ffa37b271436f8a5f89a8139044746da10c894e0a23fd6e6f070ab5358a1a9 2024-04-12T23:07:30.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 systemd[1]: Started Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: pgmap v30: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: pgmap v31: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.7 MiB/s wr, 663 op/s 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144 calling monitor election 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mon.smithi144 is new leader, mons smithi144,smithi181 in quorum (ranks 0,1) 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: monmap epoch 2 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: last_changed 2024-04-12T22:57:08.700854+0000 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: created 2024-04-12T22:54:51.314654+0000 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: min_mon_release 17 (quincy) 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: election_strategy: 1 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: 0: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-04-12T23:07:30.815 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mgrmap e34: smithi144.rlueky(active, since 54s), standbys: smithi181.whsrmz 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: mgrmap e35: smithi144.rlueky(active, since 54s), standbys: smithi181.whsrmz 2024-04-12T23:07:30.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:30 smithi144 ceph-mon[67032]: from='mgr.24501 ' entity='' 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: pgmap v30: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 687 op/s 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: pgmap v31: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.7 MiB/s wr, 663 op/s 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: mon.smithi144 calling monitor election 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: mon.smithi144 is new leader, mons smithi144,smithi181 in quorum (ranks 0,1) 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: monmap epoch 2 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: last_changed 2024-04-12T22:57:08.700854+0000 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: created 2024-04-12T22:54:51.314654+0000 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: min_mon_release 17 (quincy) 2024-04-12T23:07:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: election_strategy: 1 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: 0: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: osdmap e75: 8 total, 8 up, 8 in 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: mgrmap e34: smithi144.rlueky(active, since 54s), standbys: smithi181.whsrmz 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: mgrmap e35: smithi144.rlueky(active, since 54s), standbys: smithi181.whsrmz 2024-04-12T23:07:30.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:30 smithi181 ceph-mon[26930]: from='mgr.24501 ' entity='' 2024-04-12T23:07:31.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 150946 11.86 MB/sec execute 172 sec latency 47.036 ms 2024-04-12T23:07:32.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 151320 11.83 MB/sec execute 173 sec latency 62.492 ms 2024-04-12T23:07:32.595 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:32 smithi144 ceph-mon[67032]: mgrmap e36: smithi144.rlueky(active, since 55s), standbys: smithi181.whsrmz 2024-04-12T23:07:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:32 smithi181 ceph-mon[26930]: mgrmap e36: smithi144.rlueky(active, since 55s), standbys: smithi181.whsrmz 2024-04-12T23:07:33.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 151645 11.84 MB/sec execute 174 sec latency 62.124 ms 2024-04-12T23:07:34.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 152036 11.84 MB/sec execute 175 sec latency 37.462 ms 2024-04-12T23:07:35.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 152553 11.88 MB/sec execute 176 sec latency 43.257 ms 2024-04-12T23:07:36.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 153221 11.92 MB/sec execute 177 sec latency 17.624 ms 2024-04-12T23:07:37.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 153638 11.88 MB/sec execute 178 sec latency 39.319 ms 2024-04-12T23:07:38.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 154013 11.85 MB/sec execute 179 sec latency 63.206 ms 2024-04-12T23:07:39.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 154468 11.86 MB/sec execute 180 sec latency 57.845 ms 2024-04-12T23:07:39.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:39 smithi144 ceph-mon[67032]: Standby manager daemon smithi181.whsrmz restarted 2024-04-12T23:07:39.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:39 smithi144 ceph-mon[67032]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:07:39.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:39 smithi144 ceph-mon[67032]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:07:39.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:39 smithi144 ceph-mon[67032]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:07:39.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:39 smithi144 ceph-mon[67032]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:07:39.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:39 smithi144 ceph-mon[67032]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:07:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:39 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz restarted 2024-04-12T23:07:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:39 smithi181 ceph-mon[26930]: Standby manager daemon smithi181.whsrmz started 2024-04-12T23:07:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:39 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/crt"}]: dispatch 2024-04-12T23:07:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:39 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-12T23:07:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:39 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi181.whsrmz/key"}]: dispatch 2024-04-12T23:07:39.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:39 smithi181 ceph-mon[26930]: from='mgr.? 172.21.15.181:0/2447304313' entity='mgr.smithi181.whsrmz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-12T23:07:40.408 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:40 smithi181 ceph-mon[26930]: mgrmap e37: smithi144.rlueky(active, since 62s), standbys: smithi181.whsrmz 2024-04-12T23:07:40.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 154913 11.85 MB/sec execute 181 sec latency 45.685 ms 2024-04-12T23:07:40.455 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:40 smithi144 ceph-mon[67032]: mgrmap e37: smithi144.rlueky(active, since 62s), standbys: smithi181.whsrmz 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: Active manager daemon smithi144.rlueky restarted 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: Activating manager daemon smithi144.rlueky 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: mgrmap e38: smithi144.rlueky(active, starting, since 0.00969595s), standbys: smithi181.whsrmz 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:07:41.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: Manager daemon smithi144.rlueky is now available 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:41.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:41.222 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:07:41.222 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:07:41.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 155204 11.84 MB/sec execute 182 sec latency 39.991 ms 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: Active manager daemon smithi144.rlueky restarted 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: Activating manager daemon smithi144.rlueky 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: mgrmap e38: smithi144.rlueky(active, starting, since 0.00969595s), standbys: smithi181.whsrmz 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:07:41.413 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi144.rlueky", "id": "smithi144.rlueky"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr metadata", "who": "smithi181.whsrmz", "id": "smithi181.whsrmz"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:07:41.414 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: Manager daemon smithi144.rlueky is now available 2024-04-12T23:07:41.415 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:41.415 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:41.415 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/mirror_snapshot_schedule"}]: dispatch 2024-04-12T23:07:41.415 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:41 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi144.rlueky/trash_purge_schedule"}]: dispatch 2024-04-12T23:07:41.929 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:07:42.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 155540 11.83 MB/sec execute 183 sec latency 50.030 ms 2024-04-12T23:07:43.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:42 smithi144 ceph-mon[67032]: mgrmap e39: smithi144.rlueky(active, since 1.01696s), standbys: smithi181.whsrmz 2024-04-12T23:07:43.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:42 smithi144 ceph-mon[67032]: [12/Apr/2024:23:07:42] ENGINE Bus STARTING 2024-04-12T23:07:43.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:42 smithi144 ceph-mon[67032]: [12/Apr/2024:23:07:42] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T23:07:43.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:42 smithi144 ceph-mon[67032]: [12/Apr/2024:23:07:42] ENGINE Serving on http://172.21.15.144:8765 2024-04-12T23:07:43.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:42 smithi144 ceph-mon[67032]: [12/Apr/2024:23:07:42] ENGINE Bus STARTED 2024-04-12T23:07:43.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:42 smithi144 ceph-mon[67032]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:43.055 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:07:43.055 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (2m) 23s ago 11m 17.6M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (11m) 23s ago 11m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (10m) 27s ago 10m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 unknown 23s ago 11m - - 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (5m) 23s ago 5m 37.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (5m) 27s ago 5m 13.8M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (84s) 23s ago 12m 496M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (33s) 27s ago 10m 95.0M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (12m) 23s ago 12m 53.7M 2048M 17.2.0 e1d6a67b021e cdb5c41140d5 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (10m) 27s ago 10m 43.2M 2048M 17.2.0 e1d6a67b021e 161dc8a022e9 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (5m) 23s ago 5m 271M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (5m) 27s ago 5m 85.8M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (2m) 23s ago 11m 8464k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (95s) 27s ago 10m 8619k - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (9m) 23s ago 9m 498M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:07:43.056 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (9m) 23s ago 9m 432M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (8m) 23s ago 8m 335M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (8m) 23s ago 8m 459M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (8m) 27s ago 8m 476M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (7m) 27s ago 7m 548M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (7m) 27s ago 7m 460M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (7m) 27s ago 6m 331M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:07:43.057 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (39s) 23s ago 10m 50.1M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:07:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:42 smithi181 ceph-mon[26930]: mgrmap e39: smithi144.rlueky(active, since 1.01696s), standbys: smithi181.whsrmz 2024-04-12T23:07:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:42 smithi181 ceph-mon[26930]: [12/Apr/2024:23:07:42] ENGINE Bus STARTING 2024-04-12T23:07:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:42 smithi181 ceph-mon[26930]: [12/Apr/2024:23:07:42] ENGINE Serving on https://172.21.15.144:7150 2024-04-12T23:07:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:42 smithi181 ceph-mon[26930]: [12/Apr/2024:23:07:42] ENGINE Serving on http://172.21.15.144:8765 2024-04-12T23:07:43.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:42 smithi181 ceph-mon[26930]: [12/Apr/2024:23:07:42] ENGINE Bus STARTED 2024-04-12T23:07:43.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:42 smithi181 ceph-mon[26930]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:43.411 INFO:teuthology.orchestra.run.smithi144.stdout: 5 156086 11.88 MB/sec execute 184 sec latency 76.146 ms 2024-04-12T23:07:43.621 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:07:43.621 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:07:43.621 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T23:07:43.621 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:07:43.621 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:43.621 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:07:43.622 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:07:44.012 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:43 smithi144 ceph-mon[67032]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T23:07:44.012 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:43 smithi144 ceph-mon[67032]: Cluster is now healthy 2024-04-12T23:07:44.012 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:43 smithi144 ceph-mon[67032]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:07:44.012 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:43 smithi144 ceph-mon[67032]: mgrmap e40: smithi144.rlueky(active, since 2s), standbys: smithi181.whsrmz 2024-04-12T23:07:44.012 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:43 smithi144 ceph-mon[67032]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:44.013 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:43 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/418570458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:44.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:43 smithi181 ceph-mon[26930]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T23:07:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:43 smithi181 ceph-mon[26930]: Cluster is now healthy 2024-04-12T23:07:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:43 smithi181 ceph-mon[26930]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:07:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:43 smithi181 ceph-mon[26930]: mgrmap e40: smithi144.rlueky(active, since 2s), standbys: smithi181.whsrmz 2024-04-12T23:07:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:43 smithi181 ceph-mon[26930]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:44.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:43 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/418570458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr" 2024-04-12T23:07:44.202 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:07:44.203 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "2/23 daemons upgraded", 2024-04-12T23:07:44.203 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "", 2024-04-12T23:07:44.203 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:07:44.203 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:07:44.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 156563 11.88 MB/sec execute 185 sec latency 133.726 ms 2024-04-12T23:07:44.841 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:07:45.109 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:44 smithi144 ceph-mon[67032]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:45.109 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:44 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3228906160' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:07:45.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:44 smithi181 ceph-mon[26930]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:07:45.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:44 smithi181 ceph-mon[26930]: from='client.? 172.21.15.144:0/3228906160' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:07:45.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 156942 11.86 MB/sec execute 186 sec latency 38.340 ms 2024-04-12T23:07:45.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:45 smithi181 ceph-mon[26930]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:07:45.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:45 smithi181 ceph-mon[26930]: mgrmap e41: smithi144.rlueky(active, since 4s), standbys: smithi181.whsrmz 2024-04-12T23:07:46.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:45 smithi144 ceph-mon[67032]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:07:46.121 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:45 smithi144 ceph-mon[67032]: mgrmap e41: smithi144.rlueky(active, since 4s), standbys: smithi181.whsrmz 2024-04-12T23:07:46.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 157312 11.84 MB/sec execute 187 sec latency 38.663 ms 2024-04-12T23:07:47.392 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:47 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:47.392 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:47 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:47.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 157718 11.82 MB/sec execute 188 sec latency 55.177 ms 2024-04-12T23:07:47.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:47 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:47.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:47 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:48.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 158368 11.83 MB/sec execute 189 sec latency 56.868 ms 2024-04-12T23:07:48.597 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:48 smithi181 ceph-mon[26930]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:07:48.597 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:48 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:48.597 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:48 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:48.647 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:48 smithi144 ceph-mon[67032]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-04-12T23:07:48.647 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:48.647 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:49.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 159037 11.89 MB/sec execute 190 sec latency 55.283 ms 2024-04-12T23:07:50.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 159753 11.97 MB/sec execute 191 sec latency 53.321 ms 2024-04-12T23:07:50.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:50 smithi144 ceph-mon[67032]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 4.8 MiB/s wr, 452 op/s 2024-04-12T23:07:50.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:50 smithi181 ceph-mon[26930]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 4.8 MiB/s wr, 452 op/s 2024-04-12T23:07:51.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 160278 11.98 MB/sec execute 192 sec latency 32.430 ms 2024-04-12T23:07:52.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 160704 11.94 MB/sec execute 193 sec latency 37.343 ms 2024-04-12T23:07:52.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:52 smithi144 ceph-mon[67032]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.7 MiB/s wr, 351 op/s 2024-04-12T23:07:52.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:52.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:52.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:52 smithi181 ceph-mon[26930]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.7 MiB/s wr, 351 op/s 2024-04-12T23:07:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:52 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:52 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:53.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 161073 11.92 MB/sec execute 194 sec latency 62.423 ms 2024-04-12T23:07:53.940 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:53.940 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:53.940 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:53 smithi144 ceph-mon[67032]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 7.1 MiB/s wr, 673 op/s 2024-04-12T23:07:54.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:53 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:54.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:53 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:54.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:53 smithi181 ceph-mon[26930]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 7.1 MiB/s wr, 673 op/s 2024-04-12T23:07:54.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 161461 11.92 MB/sec execute 195 sec latency 73.792 ms 2024-04-12T23:07:55.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 161841 11.89 MB/sec execute 196 sec latency 41.201 ms 2024-04-12T23:07:55.468 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:55.468 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:55.468 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:55.468 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:07:55.468 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:55 smithi144 ceph-mon[67032]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T23:07:55.468 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:55 smithi144 ceph-mon[67032]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T23:07:55.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:55 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:55.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:55 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:55.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:55 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:55.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:55 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:07:55.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:55 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.conf 2024-04-12T23:07:55.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:55 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.conf 2024-04-12T23:07:56.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 162208 11.90 MB/sec execute 197 sec latency 44.215 ms 2024-04-12T23:07:56.501 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:07:56.501 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 6.5 MiB/s wr, 615 op/s 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:56.502 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T23:07:56.503 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.503 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 6.5 MiB/s wr, 615 op/s 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: Updating smithi144:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: Updating smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: Updating smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-12T23:07:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:56.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:56 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:07:57.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 162614 11.89 MB/sec execute 198 sec latency 47.926 ms 2024-04-12T23:07:57.517 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:57 smithi144 ceph-mon[67032]: pgmap v11: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 7.2 MiB/s wr, 683 op/s 2024-04-12T23:07:57.517 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:57 smithi144 ceph-mon[67032]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T23:07:57.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:57 smithi181 ceph-mon[26930]: pgmap v11: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 7.2 MiB/s wr, 683 op/s 2024-04-12T23:07:57.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:57 smithi181 ceph-mon[26930]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-12T23:07:58.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 162980 11.90 MB/sec execute 199 sec latency 45.013 ms 2024-04-12T23:07:58.784 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:58 smithi144 ceph-mon[67032]: Upgrade: Updating mon.smithi181 2024-04-12T23:07:58.784 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:58.784 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:07:58.785 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:07:58.785 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:58.785 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:58 smithi144 ceph-mon[67032]: Deploying daemon mon.smithi181 on smithi181 2024-04-12T23:07:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:58 smithi181 ceph-mon[26930]: Upgrade: Updating mon.smithi181 2024-04-12T23:07:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:58 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:07:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:58 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:07:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:58 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:07:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:58 smithi181 ceph-mon[26930]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:07:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:58 smithi181 ceph-mon[26930]: Deploying daemon mon.smithi181 on smithi181 2024-04-12T23:07:59.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 163365 11.92 MB/sec execute 200 sec latency 35.533 ms 2024-04-12T23:07:59.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:59 smithi181 ceph-mon[26930]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 33 KiB/s rd, 9.5 MiB/s wr, 947 op/s 2024-04-12T23:07:59.808 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:07:59 smithi144 ceph-mon[67032]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 33 KiB/s rd, 9.5 MiB/s wr, 947 op/s 2024-04-12T23:08:00.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:07:59 smithi181 systemd[1]: Stopping Ceph mon.smithi181 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:08:00.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 163859 11.92 MB/sec execute 201 sec latency 51.529 ms 2024-04-12T23:08:00.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181[26916]: 2024-04-12T23:08:00.068+0000 7fb3f3d7c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi181 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:08:00.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181[26916]: 2024-04-12T23:08:00.068+0000 7fb3f3d7c700 -1 mon.smithi181@1(peon) e2 *** Got Signal Terminated *** 2024-04-12T23:08:00.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:00 smithi181 podman[52817]: 2024-04-12 23:08:00.089803953 +0000 UTC m=+0.147242324 container died 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, name=centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., ceph=True, io.buildah.version=1.19.8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_CLEAN=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD) 2024-04-12T23:08:00.814 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:00 smithi181 podman[52817]: 2024-04-12 23:08:00.553002991 +0000 UTC m=+0.610441369 container cleanup 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, distribution-scope=public, name=centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-12T23:08:00.815 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:00 smithi181 bash[52817]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181 2024-04-12T23:08:01.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:00 smithi181 podman[52831]: 2024-04-12 23:08:00.845270971 +0000 UTC m=+0.757959435 container remove 161dc8a022e925022a2314374816eb7fca0997af4cf1367bb5a24c0acf44ff8c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, version=8, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-12T23:08:01.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi181.service: Deactivated successfully. 2024-04-12T23:08:01.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 systemd[1]: Stopped Ceph mon.smithi181 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:08:01.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi181.service: Consumed 7.339s CPU time. 2024-04-12T23:08:01.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 164427 11.90 MB/sec execute 202 sec latency 32.370 ms 2024-04-12T23:08:01.527 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 systemd[1]: Starting Ceph mon.smithi181 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:08:01.788 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 podman[52956]: 2024-04-12 23:08:01.443837086 +0000 UTC m=+0.024306875 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:08:01.789 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 podman[52956]: 2024-04-12 23:08:01.606701226 +0000 UTC m=+0.187171007 container create f55287e68062b894244fde061ece6000cbf4a5ad832d9eb660820e9511b1dca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T23:08:02.041 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 podman[52956]: 2024-04-12 23:08:01.985190192 +0000 UTC m=+0.565659977 container init f55287e68062b894244fde061ece6000cbf4a5ad832d9eb660820e9511b1dca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-12T23:08:02.041 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:01 smithi181 podman[52956]: 2024-04-12 23:08:01.989789009 +0000 UTC m=+0.570258794 container start f55287e68062b894244fde061ece6000cbf4a5ad832d9eb660820e9511b1dca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409) 2024-04-12T23:08:02.041 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: set uid:gid to 167:167 (ceph:ceph) 2024-04-12T23:08:02.041 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev), process ceph-mon, pid 2 2024-04-12T23:08:02.041 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: pidfile_write: ignore empty --pid-file 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: load: jerasure load: lrc 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: RocksDB version: 7.9.2 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Git sha 0 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Compile date 2024-04-09 21:18:54 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: DB SUMMARY 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: DB Session ID: KRZX5HYI6AFOP1XJCF08 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: CURRENT file: CURRENT 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: IDENTITY file: IDENTITY 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi181/store.db dir, Total Num: 1, files: 000024.sst 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi181/store.db: 000022.log size: 6753590 ; 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.error_if_exists: 0 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.create_if_missing: 0 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.paranoid_checks: 1 2024-04-12T23:08:02.042 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-12T23:08:02.043 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-12T23:08:02.043 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-12T23:08:02.043 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.env: 0x55c3cb0f8c60 2024-04-12T23:08:02.043 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.fs: PosixFileSystem 2024-04-12T23:08:02.043 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.info_log: 0x55c3cd9619a0 2024-04-12T23:08:02.043 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_file_opening_threads: 16 2024-04-12T23:08:02.044 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.statistics: (nil) 2024-04-12T23:08:02.044 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.use_fsync: 0 2024-04-12T23:08:02.044 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_log_file_size: 0 2024-04-12T23:08:02.044 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-12T23:08:02.044 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.keep_log_file_num: 1000 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.recycle_log_file_num: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.allow_fallocate: 1 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.allow_mmap_reads: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.allow_mmap_writes: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.use_direct_reads: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.create_missing_column_families: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.db_log_dir: 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.wal_dir: 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-12T23:08:02.045 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.advise_random_on_open: 1 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.db_write_buffer_size: 0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.write_buffer_manager: 0x55c3cd9341e0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.rate_limiter: (nil) 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.wal_recovery_mode: 2 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.enable_thread_tracking: 0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.enable_pipelined_write: 0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.unordered_write: 0 2024-04-12T23:08:02.046 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-12T23:08:02.047 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-12T23:08:02.047 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.row_cache: None 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.wal_filter: None 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.allow_ingest_behind: 0 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.two_write_queues: 0 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.manual_wal_flush: 0 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.wal_compression: 0 2024-04-12T23:08:02.048 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.atomic_flush: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.log_readahead_size: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.best_efforts_recovery: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.allow_data_in_errors: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.db_host_id: __hostname__ 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_background_jobs: 2 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_background_compactions: -1 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_subcompactions: 1 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-12T23:08:02.049 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_total_wal_size: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_open_files: -1 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bytes_per_sync: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_readahead_size: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_background_flushes: -1 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Compression algorithms supported: 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kZSTD supported: 0 2024-04-12T23:08:02.050 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kXpressCompression supported: 0 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kLZ4HCCompression supported: 1 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kZlibCompression supported: 1 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kSnappyCompression supported: 1 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kLZ4Compression supported: 1 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: kBZip2Compression supported: 0 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000009 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.merge_operator: 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_filter: None 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_filter_factory: None 2024-04-12T23:08:02.051 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.sst_partitioner_factory: None 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c3cd961aa0) 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_type: 0 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_index_type: 0 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_shortening: 1 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: checksum: 4 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: no_block_cache: 0 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache: 0x55c3cd965090 2024-04-12T23:08:02.052 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_options: 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: capacity : 536870912 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: num_shard_bits : 4 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: strict_capacity_limit : 0 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_cache_compressed: (nil) 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: persistent_cache: (nil) 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size: 4096 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_size_deviation: 10 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_restart_interval: 16 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: index_block_restart_interval: 1 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: metadata_block_size: 4096 2024-04-12T23:08:02.053 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: partition_filters: 0 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: use_delta_encoding: 1 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: filter_policy: bloomfilter 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: whole_key_filtering: 1 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: verify_compression: 0 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: format_version: 5 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_index_compression: 1 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: block_align: 0 2024-04-12T23:08:02.055 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_auto_readahead_size: 262144 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: prepopulate_block_cache: 0 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: initial_auto_readahead_size: 8192 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: num_file_reads_for_auto_readahead: 2 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.write_buffer_size: 33554432 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_write_buffer_number: 2 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression: NoCompression 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression: Disabled 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.prefix_extractor: nullptr 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.num_levels: 7 2024-04-12T23:08:02.056 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-12T23:08:02.057 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-12T23:08:02.057 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-12T23:08:02.057 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-12T23:08:02.057 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-12T23:08:02.057 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.level: 32767 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.strategy: 0 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-12T23:08:02.058 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.enabled: false 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-12T23:08:02.059 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.target_file_size_base: 67108864 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-12T23:08:02.060 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.arena_block_size: 1048576 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.disable_auto_compactions: 0 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-12T23:08:02.061 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.table_properties_collectors: 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.inplace_update_support: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.bloom_locality: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.max_successive_merges: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.paranoid_file_checks: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.force_consistency_checks: 1 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.report_bg_io_stats: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.ttl: 2592000 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-12T23:08:02.062 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.enable_blob_files: false 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.min_blob_size: 0 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.blob_file_size: 268435456 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.blob_file_starting_level: 0 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T23:08:02.063 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi181/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12236, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 93990e26-c5cb-48f7-9d10-0762aee15012 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712963282027726, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712963282044477, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4092639, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12241, "largest_seqno": 14013, "table_properties": {"data_size": 4085002, "index_size": 4826, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1797, "raw_key_size": 16323, "raw_average_key_size": 23, "raw_value_size": 4069834, "raw_average_value_size": 5764, "num_data_blocks": 219, "num_entries": 706, "num_filter_entries": 706, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712963282, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "93990e26-c5cb-48f7-9d10-0762aee15012", "db_session_id": "KRZX5HYI6AFOP1XJCF08", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712963282044587, "job": 1, "event": "recovery_finished"} 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi181/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55c3cda56000 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: DB pointer 0x55c3cda40000 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-12T23:08:02.064 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** DB Stats ** 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L0 1/0 3.90 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 506.4 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: L6 1/0 6.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Sum 2/0 10.43 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 506.4 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T23:08:02.065 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 506.4 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** Compaction Stats [default] ** 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 506.4 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-12T23:08:02.066 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Cumulative compaction: 0.00 GB write, 146.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Interval compaction: 0.00 GB write, 146.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Block cache BinnedLRUCache@0x55c3cd965090#2 capacity: 512.00 MB usage: 7.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,11.06 KB,0.00211%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: starting mon.smithi181 rank 1 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi181 fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???) e2 preinit fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).mds e7 new map 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).mds e7 print_map 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: e7 2024-04-12T23:08:02.067 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: legacy client fscid: 1 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Filesystem 'foofs' (1) 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: fs_name foofs 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: epoch 5 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: created 2024-04-12T23:02:03.352809+0000 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: modified 2024-04-12T23:02:09.376571+0000 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: tableserver 0 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: root 0 2024-04-12T23:08:02.068 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: session_timeout 60 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: session_autoclose 300 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_file_size 1099511627776 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_xattr_size 65536 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: required_client_features {} 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: last_failure 0 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: last_failure_osd_epoch 0 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: max_mds 1 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: in 0 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: up {0=14490} 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: failed 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: damaged 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: stopped 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: data_pools [3] 2024-04-12T23:08:02.069 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: metadata_pool 2 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: inline_data disabled 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: balancer 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: bal_rank_mask -1 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: standby_count_wanted 1 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: qdb_cluster leader: 0 members: 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: [mds.foofs.smithi144.alilcy{0:14490} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.144:6834/778305732,v1:172.21.15.144:6835/778305732] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: Standby daemons: 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout: [mds.foofs.smithi181.gciyns{-1:24267} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] compat {c=[1],r=[1],i=[7ff]}] 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T23:08:02.070 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T23:08:02.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-04-12T23:08:02.071 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 ceph-mon[52974]: mon.smithi181@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-04-12T23:08:02.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 165143 11.93 MB/sec execute 203 sec latency 51.818 ms 2024-04-12T23:08:02.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 bash[52956]: f55287e68062b894244fde061ece6000cbf4a5ad832d9eb660820e9511b1dca1 2024-04-12T23:08:02.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:02 smithi181 systemd[1]: Started Ceph mon.smithi181 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:08:03.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 165729 11.94 MB/sec execute 204 sec latency 36.200 ms 2024-04-12T23:08:04.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: mon.smithi144 calling monitor election 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: mon.smithi181 calling monitor election 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: mon.smithi144 is new leader, mons smithi144,smithi181 in quorum (ranks 0,1) 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: monmap epoch 3 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: last_changed 2024-04-12T23:08:03.391543+0000 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: created 2024-04-12T22:54:51.314654+0000 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: min_mon_release 19 (squid) 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: election_strategy: 1 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: 0: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:08:04.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: mgrmap e41: smithi144.rlueky(active, since 22s), standbys: smithi181.whsrmz 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:04.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:08:04.414 INFO:teuthology.orchestra.run.smithi144.stdout: 5 166248 11.98 MB/sec execute 205 sec latency 36.633 ms 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi144"}]: dispatch 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mon metadata", "id": "smithi181"}]: dispatch 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: mon.smithi144 calling monitor election 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: mon.smithi181 calling monitor election 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: mon.smithi144 is new leader, mons smithi144,smithi181 in quorum (ranks 0,1) 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: monmap epoch 3 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: fsid 9965523c-f91f-11ee-bc8d-c7b262605968 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: last_changed 2024-04-12T23:08:03.391543+0000 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: created 2024-04-12T22:54:51.314654+0000 2024-04-12T23:08:04.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: min_mon_release 19 (squid) 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: election_strategy: 1 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: 0: [v2:172.21.15.144:3300/0,v1:172.21.15.144:6789/0] mon.smithi144 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: 1: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.smithi181 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: osdmap e76: 8 total, 8 up, 8 in 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: mgrmap e41: smithi144.rlueky(active, since 22s), standbys: smithi181.whsrmz 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:04.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:08:05.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 166754 12.00 MB/sec execute 206 sec latency 31.116 ms 2024-04-12T23:08:05.434 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:05 smithi144 ceph-mon[67032]: pgmap v15: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-04-12T23:08:05.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:05 smithi181 ceph-mon[52974]: pgmap v15: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-04-12T23:08:06.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 167069 12.00 MB/sec execute 207 sec latency 44.642 ms 2024-04-12T23:08:07.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 167358 11.98 MB/sec execute 208 sec latency 67.644 ms 2024-04-12T23:08:07.482 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:07 smithi144 ceph-mon[67032]: pgmap v16: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-04-12T23:08:07.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:07 smithi181 ceph-mon[52974]: pgmap v16: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 590 op/s 2024-04-12T23:08:08.412 INFO:teuthology.orchestra.run.smithi144.stdout: 5 167585 11.93 MB/sec execute 209 sec latency 80.301 ms 2024-04-12T23:08:08.753 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:08.753 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:08.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:09.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 167858 11.89 MB/sec execute 210 sec latency 40.602 ms 2024-04-12T23:08:09.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:09 smithi144 ceph-mon[67032]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 819 op/s 2024-04-12T23:08:09.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:09 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:09.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:09 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:09.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:09 smithi181 ceph-mon[52974]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 819 op/s 2024-04-12T23:08:09.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:09 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:09.924 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:09 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:10.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 168117 11.87 MB/sec execute 211 sec latency 40.774 ms 2024-04-12T23:08:11.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 168476 11.86 MB/sec execute 212 sec latency 54.647 ms 2024-04-12T23:08:11.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:11 smithi144 ceph-mon[67032]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 581 op/s 2024-04-12T23:08:11.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:11.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:11.944 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:11 smithi181 ceph-mon[52974]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 581 op/s 2024-04-12T23:08:11.944 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:11.944 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:12.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 168859 11.83 MB/sec execute 213 sec latency 85.639 ms 2024-04-12T23:08:13.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 169184 11.83 MB/sec execute 214 sec latency 34.007 ms 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 581 op/s 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.659 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.660 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:08:13.660 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:08:13.660 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.3 MiB/s wr, 581 op/s 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:08:13.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:14.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 169453 11.80 MB/sec execute 215 sec latency 53.289 ms 2024-04-12T23:08:14.682 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:14 smithi144 ceph-mon[67032]: pgmap v20: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 846 op/s 2024-04-12T23:08:14.682 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:14 smithi144 ceph-mon[67032]: Reconfiguring mon.smithi144 (monmap changed)... 2024-04-12T23:08:14.682 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:14 smithi144 ceph-mon[67032]: Reconfiguring daemon mon.smithi144 on smithi144 2024-04-12T23:08:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:14 smithi181 ceph-mon[52974]: pgmap v20: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 846 op/s 2024-04-12T23:08:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:14 smithi181 ceph-mon[52974]: Reconfiguring mon.smithi144 (monmap changed)... 2024-04-12T23:08:14.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:14 smithi181 ceph-mon[52974]: Reconfiguring daemon mon.smithi144 on smithi144 2024-04-12T23:08:15.365 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:08:15.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 169774 11.81 MB/sec execute 216 sec latency 70.582 ms 2024-04-12T23:08:16.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 523 op/s 2024-04-12T23:08:16.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:16.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:16.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: Reconfiguring mgr.smithi144.rlueky (monmap changed)... 2024-04-12T23:08:16.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:08:16.214 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:08:16.215 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:16.215 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: Reconfiguring daemon mgr.smithi144.rlueky on smithi144 2024-04-12T23:08:16.215 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:16 smithi144 ceph-mon[67032]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:16.309 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (3m) 23s ago 11m 17.6M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (11m) 23s ago 11m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (11m) 8s ago 11m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 unknown 23s ago 11m - - 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (6m) 23s ago 6m 37.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (6m) 8s ago 6m 14.2M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (117s) 23s ago 13m 479M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (66s) 8s ago 11m 448M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (46s) 23s ago 13m 32.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (14s) 8s ago 11m 23.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (5m) 23s ago 5m 275M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (5m) 8s ago 5m 88.3M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:08:16.310 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (2m) 23s ago 11m 10.6M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (2m) 8s ago 10m 17.1M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (10m) 23s ago 10m 549M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (9m) 23s ago 9m 475M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (9m) 23s ago 9m 378M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (9m) 23s ago 9m 490M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (8m) 8s ago 8m 527M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (8m) 8s ago 8m 360M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (7m) 8s ago 7m 541M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (7m) 8s ago 7m 421M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:08:16.311 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (73s) 23s ago 10m 54.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:08:16.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 170038 11.79 MB/sec execute 217 sec latency 51.912 ms 2024-04-12T23:08:16.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 523 op/s 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: Reconfiguring mgr.smithi144.rlueky (monmap changed)... 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi144.rlueky", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: Reconfiguring daemon mgr.smithi144.rlueky on smithi144 2024-04-12T23:08:16.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:16 smithi181 ceph-mon[52974]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:16.873 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:08:16.874 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:08:17.230 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:17 smithi144 ceph-mon[67032]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:17.230 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:17 smithi144 ceph-mon[67032]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:17.230 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:17 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/793890452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:17.358 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "4/23 daemons upgraded", 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading mon daemons", 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:08:17.369 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:08:17.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 170378 11.80 MB/sec execute 218 sec latency 43.519 ms 2024-04-12T23:08:17.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:17 smithi181 ceph-mon[52974]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:17.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:17 smithi181 ceph-mon[52974]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:17.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:17 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/793890452' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:17.968 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:08:17.968 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:08:17.968 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:08:18.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 523 op/s 2024-04-12T23:08:18.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:18.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:18.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:18.250 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3448526532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:08:18.251 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:08:18.251 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:18.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 170609 11.78 MB/sec execute 219 sec latency 64.054 ms 2024-04-12T23:08:18.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.1 MiB/s wr, 523 op/s 2024-04-12T23:08:18.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:18.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:18.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:18.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3448526532' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:08:18.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:08:18.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:19.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:19 smithi144 ceph-mon[67032]: Reconfiguring crash.smithi144 (monmap changed)... 2024-04-12T23:08:19.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:19 smithi144 ceph-mon[67032]: Reconfiguring daemon crash.smithi144 on smithi144 2024-04-12T23:08:19.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 170925 11.76 MB/sec execute 220 sec latency 44.273 ms 2024-04-12T23:08:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:19 smithi181 ceph-mon[52974]: Reconfiguring crash.smithi144 (monmap changed)... 2024-04-12T23:08:19.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:19 smithi181 ceph-mon[52974]: Reconfiguring daemon crash.smithi144 on smithi144 2024-04-12T23:08:20.297 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:20 smithi144 ceph-mon[67032]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 399 op/s 2024-04-12T23:08:20.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 171233 11.72 MB/sec execute 221 sec latency 46.156 ms 2024-04-12T23:08:20.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:20 smithi181 ceph-mon[52974]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 399 op/s 2024-04-12T23:08:21.414 INFO:teuthology.orchestra.run.smithi144.stdout: 5 171438 11.68 MB/sec execute 222 sec latency 56.879 ms 2024-04-12T23:08:21.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:21 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:21.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:21 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:21.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:21 smithi144 ceph-mon[67032]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T23:08:21.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:21 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T23:08:21.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:21 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:21.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:21 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.0 on smithi144 2024-04-12T23:08:21.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:21 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:21.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:21 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:21.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:21 smithi181 ceph-mon[52974]: Reconfiguring osd.0 (monmap changed)... 2024-04-12T23:08:21.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:21 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T23:08:21.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:21 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:21.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:21 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.0 on smithi144 2024-04-12T23:08:22.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 171680 11.66 MB/sec execute 223 sec latency 57.224 ms 2024-04-12T23:08:22.619 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:22 smithi144 ceph-mon[67032]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 399 op/s 2024-04-12T23:08:22.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:22 smithi181 ceph-mon[52974]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 399 op/s 2024-04-12T23:08:23.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 172025 11.65 MB/sec execute 224 sec latency 82.349 ms 2024-04-12T23:08:24.413 INFO:teuthology.orchestra.run.smithi144.stdout: 5 172415 11.62 MB/sec execute 225 sec latency 73.474 ms 2024-04-12T23:08:24.657 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:24 smithi144 ceph-mon[67032]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-04-12T23:08:24.657 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:24 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:24.657 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:24 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:24.657 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:24 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T23:08:24.657 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:24 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:24.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:24 smithi181 ceph-mon[52974]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-04-12T23:08:24.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:24 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:24 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:24 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T23:08:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:24 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:25.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 172833 11.63 MB/sec execute 226 sec latency 32.157 ms 2024-04-12T23:08:25.675 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:25 smithi144 ceph-mon[67032]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T23:08:25.675 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:25 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.1 on smithi144 2024-04-12T23:08:25.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:25 smithi181 ceph-mon[52974]: Reconfiguring osd.1 (monmap changed)... 2024-04-12T23:08:25.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:25 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.1 on smithi144 2024-04-12T23:08:26.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 173287 11.65 MB/sec execute 227 sec latency 34.203 ms 2024-04-12T23:08:26.689 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:26 smithi144 ceph-mon[67032]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s 2024-04-12T23:08:26.689 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:26 smithi181 ceph-mon[52974]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s 2024-04-12T23:08:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:27.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 173691 11.65 MB/sec execute 228 sec latency 29.573 ms 2024-04-12T23:08:28.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 173975 11.66 MB/sec execute 229 sec latency 68.446 ms 2024-04-12T23:08:28.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: pgmap v27: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s 2024-04-12T23:08:28.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:28.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:28.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T23:08:28.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T23:08:28.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:28.717 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:28 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.2 on smithi144 2024-04-12T23:08:28.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: pgmap v27: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 360 op/s 2024-04-12T23:08:28.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:28.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:28.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: Reconfiguring osd.2 (monmap changed)... 2024-04-12T23:08:28.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T23:08:28.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:28.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:28 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.2 on smithi144 2024-04-12T23:08:29.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 174225 11.63 MB/sec execute 230 sec latency 65.871 ms 2024-04-12T23:08:30.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 174501 11.61 MB/sec execute 231 sec latency 61.273 ms 2024-04-12T23:08:30.745 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:30 smithi144 ceph-mon[67032]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-04-12T23:08:30.745 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:30 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:30.745 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:30 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:30.746 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:30 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T23:08:30.746 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:30 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:30 smithi181 ceph-mon[52974]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-04-12T23:08:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:30 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:30 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:30 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T23:08:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:30 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:31.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 174789 11.57 MB/sec execute 232 sec latency 65.933 ms 2024-04-12T23:08:31.760 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:31 smithi144 ceph-mon[67032]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T23:08:31.760 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:31 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.3 on smithi144 2024-04-12T23:08:31.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:31 smithi181 ceph-mon[52974]: Reconfiguring osd.3 (monmap changed)... 2024-04-12T23:08:31.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:31 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.3 on smithi144 2024-04-12T23:08:32.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 175050 11.55 MB/sec execute 233 sec latency 33.018 ms 2024-04-12T23:08:32.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:32 smithi144 ceph-mon[67032]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-04-12T23:08:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:32 smithi181 ceph-mon[52974]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-04-12T23:08:33.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 175330 11.53 MB/sec execute 234 sec latency 61.992 ms 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: Reconfiguring mds.foofs.smithi144.alilcy (monmap changed)... 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:34.293 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:34 smithi144 ceph-mon[67032]: Reconfiguring daemon mds.foofs.smithi144.alilcy on smithi144 2024-04-12T23:08:34.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 175732 11.53 MB/sec execute 235 sec latency 60.432 ms 2024-04-12T23:08:34.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-04-12T23:08:34.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: Reconfiguring mds.foofs.smithi144.alilcy (monmap changed)... 2024-04-12T23:08:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:08:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:34 smithi181 ceph-mon[52974]: Reconfiguring daemon mds.foofs.smithi144.alilcy on smithi144 2024-04-12T23:08:35.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 176291 11.53 MB/sec execute 236 sec latency 39.457 ms 2024-04-12T23:08:36.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 176796 11.55 MB/sec execute 237 sec latency 53.364 ms 2024-04-12T23:08:36.599 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 404 op/s 2024-04-12T23:08:36.599 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:36.599 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:36.599 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: Reconfiguring crash.smithi181 (monmap changed)... 2024-04-12T23:08:36.599 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:08:36.599 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:36.600 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:36 smithi144 ceph-mon[67032]: Reconfiguring daemon crash.smithi181 on smithi181 2024-04-12T23:08:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 404 op/s 2024-04-12T23:08:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: Reconfiguring crash.smithi181 (monmap changed)... 2024-04-12T23:08:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:08:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:36 smithi181 ceph-mon[52974]: Reconfiguring daemon crash.smithi181 on smithi181 2024-04-12T23:08:37.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 177204 11.56 MB/sec execute 238 sec latency 40.374 ms 2024-04-12T23:08:38.415 INFO:teuthology.orchestra.run.smithi144.stdout: 5 177534 11.57 MB/sec execute 239 sec latency 42.602 ms 2024-04-12T23:08:38.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 404 op/s 2024-04-12T23:08:38.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:38.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:38.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: Reconfiguring mgr.smithi181.whsrmz (monmap changed)... 2024-04-12T23:08:38.633 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:08:38.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:08:38.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:38.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:38 smithi144 ceph-mon[67032]: Reconfiguring daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T23:08:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.1 MiB/s wr, 404 op/s 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: Reconfiguring mgr.smithi181.whsrmz (monmap changed)... 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi181.whsrmz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:38.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:38 smithi181 ceph-mon[52974]: Reconfiguring daemon mgr.smithi181.whsrmz on smithi181 2024-04-12T23:08:39.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 177801 11.55 MB/sec execute 240 sec latency 49.447 ms 2024-04-12T23:08:40.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: pgmap v33: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 647 op/s 2024-04-12T23:08:40.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:40.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:40.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: Reconfiguring mon.smithi181 (monmap changed)... 2024-04-12T23:08:40.365 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:08:40.365 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:08:40.366 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:40.366 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:40 smithi181 ceph-mon[52974]: Reconfiguring daemon mon.smithi181 on smithi181 2024-04-12T23:08:40.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 178215 11.53 MB/sec execute 241 sec latency 62.305 ms 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: pgmap v33: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 647 op/s 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: Reconfiguring mon.smithi181 (monmap changed)... 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:40.416 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:40 smithi144 ceph-mon[67032]: Reconfiguring daemon mon.smithi181 on smithi181 2024-04-12T23:08:41.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 178568 11.51 MB/sec execute 242 sec latency 32.289 ms 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: pgmap v34: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.4 on smithi181 2024-04-12T23:08:41.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:42.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: pgmap v34: 97 pgs: 97 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: Reconfiguring osd.4 (monmap changed)... 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.4 on smithi181 2024-04-12T23:08:42.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:42.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 179025 11.53 MB/sec execute 243 sec latency 42.975 ms 2024-04-12T23:08:43.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 179598 11.51 MB/sec execute 244 sec latency 57.076 ms 2024-04-12T23:08:44.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 180090 11.53 MB/sec execute 245 sec latency 29.006 ms 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: pgmap v35: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 656 op/s 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:44 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.5 on smithi181 2024-04-12T23:08:44.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: pgmap v35: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 656 op/s 2024-04-12T23:08:44.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:44.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:44.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: Reconfiguring osd.5 (monmap changed)... 2024-04-12T23:08:44.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T23:08:44.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:44.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:44 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.5 on smithi181 2024-04-12T23:08:45.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 180646 11.55 MB/sec execute 246 sec latency 30.823 ms 2024-04-12T23:08:46.219 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: pgmap v36: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 471 op/s 2024-04-12T23:08:46.219 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:46.219 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:46.219 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T23:08:46.219 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T23:08:46.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:46.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:46 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.6 on smithi181 2024-04-12T23:08:46.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 181091 11.58 MB/sec execute 247 sec latency 51.771 ms 2024-04-12T23:08:46.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: pgmap v36: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 471 op/s 2024-04-12T23:08:46.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:46.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:46.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: Reconfiguring osd.6 (monmap changed)... 2024-04-12T23:08:46.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T23:08:46.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:46.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:46 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.6 on smithi181 2024-04-12T23:08:47.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 181513 11.58 MB/sec execute 248 sec latency 65.036 ms 2024-04-12T23:08:48.243 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: pgmap v37: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 471 op/s 2024-04-12T23:08:48.243 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:48.243 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:48.243 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T23:08:48.243 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T23:08:48.243 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:48.244 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:48 smithi144 ceph-mon[67032]: Reconfiguring daemon osd.7 on smithi181 2024-04-12T23:08:48.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 182046 11.56 MB/sec execute 249 sec latency 55.619 ms 2024-04-12T23:08:48.554 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:08:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: pgmap v37: 97 pgs: 97 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 471 op/s 2024-04-12T23:08:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: Reconfiguring osd.7 (monmap changed)... 2024-04-12T23:08:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T23:08:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:48.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:48 smithi181 ceph-mon[52974]: Reconfiguring daemon osd.7 on smithi181 2024-04-12T23:08:49.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:49 smithi144 ceph-mon[67032]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:49.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 182676 11.59 MB/sec execute 250 sec latency 37.569 ms 2024-04-12T23:08:49.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:49 smithi181 ceph-mon[52974]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (3m) 56s ago 12m 17.6M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (12m) 56s ago 12m 6405k - 17.2.0 e1d6a67b021e 13529116c20a 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (11m) 42s ago 11m 6400k - 17.2.0 e1d6a67b021e 5c72c01859d1 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 unknown 56s ago 12m - - 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (6m) 56s ago 6m 37.9M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (6m) 42s ago 6m 14.2M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (2m) 56s ago 13m 479M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (100s) 42s ago 11m 448M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (79s) 56s ago 13m 32.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:08:49.683 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (48s) 42s ago 11m 23.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (6m) 56s ago 6m 275M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (6m) 42s ago 6m 88.3M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (3m) 56s ago 12m 10.6M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (2m) 42s ago 11m 17.1M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (11m) 56s ago 11m 549M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (10m) 56s ago 10m 475M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (10m) 56s ago 10m 378M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (9m) 56s ago 9m 490M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (9m) 42s ago 9m 527M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (8m) 42s ago 8m 360M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (8m) 42s ago 8m 541M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (8m) 42s ago 8m 421M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:08:49.684 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (106s) 56s ago 11m 54.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:08:50.263 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: pgmap v38: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 765 op/s 2024-04-12T23:08:50.263 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:50.263 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:50.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:50.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: Reconfiguring mds.foofs.smithi181.gciyns (monmap changed)... 2024-04-12T23:08:50.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:08:50.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:50.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: Reconfiguring daemon mds.foofs.smithi181.gciyns on smithi181 2024-04-12T23:08:50.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:50 smithi144 ceph-mon[67032]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:08:50.399 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:08:50.400 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:08:50.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 183410 11.61 MB/sec execute 251 sec latency 35.324 ms 2024-04-12T23:08:50.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: pgmap v38: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 765 op/s 2024-04-12T23:08:50.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: Reconfiguring mds.foofs.smithi181.gciyns (monmap changed)... 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: Reconfiguring daemon mds.foofs.smithi181.gciyns on smithi181 2024-04-12T23:08:50.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:50 smithi181 ceph-mon[52974]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:50.951 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "4/23 daemons upgraded", 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading mon daemons", 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:08:50.952 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:08:51.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2668374093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:51.269 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"}]: dispatch 2024-04-12T23:08:51.270 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"}]': finished 2024-04-12T23:08:51.271 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]: dispatch 2024-04-12T23:08:51.271 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]': finished 2024-04-12T23:08:51.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 183847 11.61 MB/sec execute 252 sec latency 63.957 ms 2024-04-12T23:08:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2668374093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"}]: dispatch 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi144"}]': finished 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]: dispatch 2024-04-12T23:08:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi181"}]': finished 2024-04-12T23:08:51.570 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:08:51.570 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:08:51.570 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in unknown state 2024-04-12T23:08:52.284 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:52 smithi144 ceph-mon[67032]: pgmap v39: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 522 op/s 2024-04-12T23:08:52.284 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:52 smithi144 ceph-mon[67032]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:52.284 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:52 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all mon 2024-04-12T23:08:52.284 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:52 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1089801462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:08:52.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 184143 11.60 MB/sec execute 253 sec latency 48.392 ms 2024-04-12T23:08:52.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:52 smithi181 ceph-mon[52974]: pgmap v39: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 522 op/s 2024-04-12T23:08:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:52 smithi181 ceph-mon[52974]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:08:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:52 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all mon 2024-04-12T23:08:52.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:52 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1089801462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:08:53.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 184464 11.61 MB/sec execute 254 sec latency 65.847 ms 2024-04-12T23:08:54.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:54 smithi144 ceph-mon[67032]: pgmap v40: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 852 op/s 2024-04-12T23:08:54.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:54.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:08:54.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:54.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 184690 11.60 MB/sec execute 255 sec latency 58.091 ms 2024-04-12T23:08:54.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:54 smithi181 ceph-mon[52974]: pgmap v40: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 852 op/s 2024-04-12T23:08:54.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:08:54.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi144", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:08:54.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:08:55.362 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:55 smithi144 ceph-mon[67032]: Upgrade: Updating crash.smithi144 (1/2) 2024-04-12T23:08:55.362 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:55 smithi144 ceph-mon[67032]: Deploying daemon crash.smithi144 on smithi144 2024-04-12T23:08:55.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 185099 11.60 MB/sec execute 256 sec latency 34.475 ms 2024-04-12T23:08:55.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:55 smithi181 ceph-mon[52974]: Upgrade: Updating crash.smithi144 (1/2) 2024-04-12T23:08:55.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:55 smithi181 ceph-mon[52974]: Deploying daemon crash.smithi144 on smithi144 2024-04-12T23:08:56.380 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:56 smithi144 ceph-mon[67032]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 623 op/s 2024-04-12T23:08:56.380 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:56.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 185571 11.58 MB/sec execute 257 sec latency 34.309 ms 2024-04-12T23:08:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:56 smithi181 ceph-mon[52974]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 623 op/s 2024-04-12T23:08:56.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:08:57.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 185878 11.56 MB/sec execute 258 sec latency 41.219 ms 2024-04-12T23:08:58.409 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:08:58 smithi144 ceph-mon[67032]: pgmap v42: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 623 op/s 2024-04-12T23:08:58.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 186207 11.56 MB/sec execute 259 sec latency 91.361 ms 2024-04-12T23:08:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:08:58 smithi181 ceph-mon[52974]: pgmap v42: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 623 op/s 2024-04-12T23:08:59.416 INFO:teuthology.orchestra.run.smithi144.stdout: 5 186617 11.53 MB/sec execute 260 sec latency 70.992 ms 2024-04-12T23:09:00.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 186946 11.53 MB/sec execute 261 sec latency 45.736 ms 2024-04-12T23:09:00.461 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:00 smithi144 ceph-mon[67032]: pgmap v43: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 843 op/s 2024-04-12T23:09:00.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:00 smithi181 ceph-mon[52974]: pgmap v43: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 843 op/s 2024-04-12T23:09:01.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 187455 11.55 MB/sec execute 262 sec latency 45.055 ms 2024-04-12T23:09:01.977 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:01 smithi144 ceph-mon[67032]: pgmap v44: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 550 op/s 2024-04-12T23:09:01.977 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:01 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:01.977 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:01 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:02.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:01 smithi181 ceph-mon[52974]: pgmap v44: 97 pgs: 97 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 550 op/s 2024-04-12T23:09:02.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:01 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:02.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:01 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:02.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 188024 11.58 MB/sec execute 263 sec latency 25.926 ms 2024-04-12T23:09:03.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 188430 11.58 MB/sec execute 264 sec latency 33.031 ms 2024-04-12T23:09:04.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:03 smithi144 ceph-mon[67032]: pgmap v45: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 786 op/s 2024-04-12T23:09:04.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:03 smithi144 ceph-mon[67032]: Upgrade: Updating crash.smithi181 (2/2) 2024-04-12T23:09:04.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:03 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:04.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:03 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:09:04.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:03 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:04.003 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:03 smithi144 ceph-mon[67032]: Deploying daemon crash.smithi181 on smithi181 2024-04-12T23:09:04.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:03 smithi181 ceph-mon[52974]: pgmap v45: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 786 op/s 2024-04-12T23:09:04.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:03 smithi181 ceph-mon[52974]: Upgrade: Updating crash.smithi181 (2/2) 2024-04-12T23:09:04.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:03 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:04.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:03 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi181", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-12T23:09:04.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:03 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:04.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:03 smithi181 ceph-mon[52974]: Deploying daemon crash.smithi181 on smithi181 2024-04-12T23:09:04.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 188904 11.57 MB/sec execute 265 sec latency 39.195 ms 2024-04-12T23:09:05.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 189279 11.55 MB/sec execute 266 sec latency 41.425 ms 2024-04-12T23:09:06.044 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:05 smithi144 ceph-mon[67032]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-04-12T23:09:06.103 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:05 smithi181 ceph-mon[52974]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-04-12T23:09:06.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 189797 11.57 MB/sec execute 267 sec latency 47.429 ms 2024-04-12T23:09:07.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 190392 11.58 MB/sec execute 268 sec latency 50.633 ms 2024-04-12T23:09:08.369 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:08 smithi144 ceph-mon[67032]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-04-12T23:09:08.369 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:08.369 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:08.369 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:08.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 190545 11.55 MB/sec execute 269 sec latency 74.122 ms 2024-04-12T23:09:08.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:08 smithi181 ceph-mon[52974]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 456 op/s 2024-04-12T23:09:08.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:08.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:08.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:09.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 190778 11.53 MB/sec execute 270 sec latency 75.497 ms 2024-04-12T23:09:10.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 191068 11.53 MB/sec execute 271 sec latency 50.295 ms 2024-04-12T23:09:10.424 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:10 smithi144 ceph-mon[67032]: pgmap v48: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 752 op/s 2024-04-12T23:09:10.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:10 smithi181 ceph-mon[52974]: pgmap v48: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 752 op/s 2024-04-12T23:09:11.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 191367 11.51 MB/sec execute 272 sec latency 76.761 ms 2024-04-12T23:09:11.433 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:11.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:12.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 191705 11.53 MB/sec execute 273 sec latency 38.165 ms 2024-04-12T23:09:12.710 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:12 smithi144 ceph-mon[67032]: pgmap v49: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 533 op/s 2024-04-12T23:09:12.710 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:12 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:12.710 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:12 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:12.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:12 smithi181 ceph-mon[52974]: pgmap v49: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 533 op/s 2024-04-12T23:09:12.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:12 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:12.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:12 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:13.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 191937 11.51 MB/sec execute 274 sec latency 68.962 ms 2024-04-12T23:09:13.974 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:13 smithi144 ceph-mon[67032]: pgmap v50: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 702 op/s 2024-04-12T23:09:13.974 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:13.974 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:14.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:13 smithi181 ceph-mon[52974]: pgmap v50: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 702 op/s 2024-04-12T23:09:14.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:14.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:14.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 192197 11.49 MB/sec execute 275 sec latency 62.467 ms 2024-04-12T23:09:14.987 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:14 smithi144 ceph-mon[67032]: mgrmap e42: smithi144.rlueky(active, since 92s), standbys: smithi181.whsrmz 2024-04-12T23:09:15.142 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:14 smithi181 ceph-mon[52974]: mgrmap e42: smithi144.rlueky(active, since 92s), standbys: smithi181.whsrmz 2024-04-12T23:09:15.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 192514 11.46 MB/sec execute 276 sec latency 64.797 ms 2024-04-12T23:09:15.999 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:15 smithi144 ceph-mon[67032]: pgmap v51: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 466 op/s 2024-04-12T23:09:16.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:15 smithi181 ceph-mon[52974]: pgmap v51: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 466 op/s 2024-04-12T23:09:16.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 192827 11.44 MB/sec execute 277 sec latency 34.780 ms 2024-04-12T23:09:17.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 193210 11.46 MB/sec execute 278 sec latency 59.096 ms 2024-04-12T23:09:17.541 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.541 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.541 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.541 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:17.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:18.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 193616 11.43 MB/sec execute 279 sec latency 87.190 ms 2024-04-12T23:09:18.805 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:18 smithi144 ceph-mon[67032]: pgmap v52: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 466 op/s 2024-04-12T23:09:18.805 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:18.805 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:18 smithi181 ceph-mon[52974]: pgmap v52: 97 pgs: 97 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 466 op/s 2024-04-12T23:09:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:18.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:19.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 194002 11.43 MB/sec execute 280 sec latency 42.910 ms 2024-04-12T23:09:20.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 194233 11.40 MB/sec execute 281 sec latency 60.130 ms 2024-04-12T23:09:20.863 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:20 smithi144 ceph-mon[67032]: pgmap v53: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 662 op/s 2024-04-12T23:09:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:20 smithi181 ceph-mon[52974]: pgmap v53: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 662 op/s 2024-04-12T23:09:21.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 194663 11.42 MB/sec execute 282 sec latency 41.943 ms 2024-04-12T23:09:22.077 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:09:22.417 INFO:teuthology.orchestra.run.smithi144.stdout: 5 195143 11.45 MB/sec execute 283 sec latency 43.329 ms 2024-04-12T23:09:22.897 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:22 smithi144 ceph-mon[67032]: pgmap v54: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 366 op/s 2024-04-12T23:09:22.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:22 smithi181 ceph-mon[52974]: pgmap v54: 97 pgs: 97 active+clean; 336 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 366 op/s 2024-04-12T23:09:23.136 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (4m) 6s ago 13m 17.6M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (23s) 6s ago 12m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (16s) 11s ago 12m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 6s ago 12m - - 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (7m) 6s ago 7m 38.3M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (7m) 11s ago 7m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (3m) 6s ago 14m 501M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (2m) 11s ago 12m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (113s) 6s ago 14m 43.6M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (81s) 11s ago 12m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (6m) 6s ago 6m 269M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (6m) 11s ago 6m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:09:23.137 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (3m) 6s ago 12m 9239k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (3m) 11s ago 12m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (11m) 6s ago 11m 392M 1936M 17.2.0 e1d6a67b021e 8dabaea4a775 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (11m) 6s ago 11m 570M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (10m) 6s ago 10m 491M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (10m) 6s ago 10m 591M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (9m) 11s ago 9m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (9m) 11s ago 9m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (9m) 11s ago 9m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (8m) 11s ago 8m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:09:23.138 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (2m) 6s ago 11m 60.3M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:09:23.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 195524 11.45 MB/sec execute 284 sec latency 42.772 ms 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-12T23:09:23.899 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:09:23.900 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:09:23.916 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"}]: dispatch 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"}]': finished 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]: dispatch 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]': finished 2024-04-12T23:09:23.917 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:23 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T23:09:23.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:23.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:23.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi144"}]': finished 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]: dispatch 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi181"}]': finished 2024-04-12T23:09:23.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:23 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "6/23 daemons upgraded", 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:09:24.380 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:09:24.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 195900 11.43 MB/sec execute 285 sec latency 49.019 ms 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: pgmap v55: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 579 op/s 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: pgmap v56: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 482 op/s 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: pgmap v57: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 600 op/s 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all crash 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: Upgrade: osd.0 is safe to restart 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:24.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:24 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/48006697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: pgmap v55: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.0 MiB/s wr, 579 op/s 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: pgmap v56: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 482 op/s 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: pgmap v57: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 600 op/s 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all crash 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: Upgrade: osd.0 is safe to restart 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:24.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:24 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/48006697' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:24.969 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:09:24.969 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:09:24.969 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:09:25.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 196225 11.40 MB/sec execute 286 sec latency 49.847 ms 2024-04-12T23:09:25.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:25 smithi144 ceph-mon[67032]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:25.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:25 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2872501092' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:09:25.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:25 smithi181 ceph-mon[52974]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:25.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:25 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2872501092' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:09:26.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 196531 11.39 MB/sec execute 287 sec latency 68.433 ms 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: pgmap v58: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 600 op/s 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: Upgrade: Updating osd.0 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:26.706 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:26 smithi144 ceph-mon[67032]: Deploying daemon osd.0 on smithi144 2024-04-12T23:09:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: pgmap v58: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 600 op/s 2024-04-12T23:09:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: Upgrade: Updating osd.0 2024-04-12T23:09:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-12T23:09:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:26.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:26 smithi181 ceph-mon[52974]: Deploying daemon osd.0 on smithi144 2024-04-12T23:09:27.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 196951 11.40 MB/sec execute 288 sec latency 57.643 ms 2024-04-12T23:09:27.718 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:27 smithi144 ceph-mon[67032]: pgmap v59: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.2 MiB/s wr, 312 op/s 2024-04-12T23:09:27.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:27 smithi181 ceph-mon[52974]: pgmap v59: 97 pgs: 97 active+clean; 339 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.2 MiB/s wr, 312 op/s 2024-04-12T23:09:28.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 197491 11.40 MB/sec execute 289 sec latency 88.424 ms 2024-04-12T23:09:29.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 197810 11.38 MB/sec execute 290 sec latency 45.528 ms 2024-04-12T23:09:30.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:29 smithi144 ceph-mon[67032]: pgmap v60: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 668 op/s 2024-04-12T23:09:30.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:29 smithi181 ceph-mon[52974]: pgmap v60: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 668 op/s 2024-04-12T23:09:30.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 198298 11.40 MB/sec execute 291 sec latency 37.262 ms 2024-04-12T23:09:31.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 198622 11.41 MB/sec execute 292 sec latency 40.662 ms 2024-04-12T23:09:31.789 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:31 smithi144 systemd[1]: Stopping Ceph osd.0 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:09:32.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:31 smithi181 ceph-mon[52974]: pgmap v61: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 356 op/s 2024-04-12T23:09:32.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:31 smithi181 ceph-mon[52974]: osd.0 marked itself down and dead 2024-04-12T23:09:32.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:31 smithi144 ceph-mon[67032]: pgmap v61: 97 pgs: 97 active+clean; 335 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 356 op/s 2024-04-12T23:09:32.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:31 smithi144 ceph-mon[67032]: osd.0 marked itself down and dead 2024-04-12T23:09:32.274 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:31 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[35281]: 2024-04-12T23:09:31.944+0000 7f7b4ba92700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:09:32.274 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:31 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[35281]: 2024-04-12T23:09:31.944+0000 7f7b4ba92700 -1 osd.0 76 *** Got signal Terminated *** 2024-04-12T23:09:32.274 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:31 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[35281]: 2024-04-12T23:09:31.944+0000 7f7b4ba92700 -1 osd.0 76 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:09:32.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 198786 11.39 MB/sec execute 293 sec latency 494.840 ms 2024-04-12T23:09:32.888 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:32 smithi144 podman[73053]: 2024-04-12 23:09:32.617781778 +0000 UTC m=+0.805247910 container died 8dabaea4a7750bd3fc97c7bce166579ea3d9761f533b002873f06c40d479815f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_CLEAN=True, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-12T23:09:33.139 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:32 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:09:33.139 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:32 smithi144 ceph-mon[67032]: osdmap e77: 8 total, 7 up, 8 in 2024-04-12T23:09:33.139 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:33 smithi144 podman[73053]: 2024-04-12 23:09:33.016072235 +0000 UTC m=+1.203538370 container cleanup 8dabaea4a7750bd3fc97c7bce166579ea3d9761f533b002873f06c40d479815f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container) 2024-04-12T23:09:33.139 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:33 smithi144 bash[73053]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0 2024-04-12T23:09:33.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:32 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:09:33.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:32 smithi181 ceph-mon[52974]: osdmap e77: 8 total, 7 up, 8 in 2024-04-12T23:09:33.400 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:33 smithi144 podman[73072]: 2024-04-12 23:09:33.351433123 +0000 UTC m=+0.725993919 container remove 8dabaea4a7750bd3fc97c7bce166579ea3d9761f533b002873f06c40d479815f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, release=754, architecture=x86_64, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T23:09:33.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 198869 11.36 MB/sec execute 294 sec latency 1078.454 ms 2024-04-12T23:09:33.908 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:33 smithi144 podman[73158]: 2024-04-12 23:09:33.773569218 +0000 UTC m=+0.020337632 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:09:33.908 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:33 smithi144 podman[73158]: 2024-04-12 23:09:33.894096035 +0000 UTC m=+0.140864427 container create 8815b9be6d25536a08a8e71e8797c0ef453367d5e766d3129ff63691604dcb7c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb) 2024-04-12T23:09:34.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:33 smithi144 ceph-mon[67032]: pgmap v63: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-04-12T23:09:34.159 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:33 smithi144 ceph-mon[67032]: osdmap e78: 8 total, 7 up, 8 in 2024-04-12T23:09:34.416 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:34 smithi144 podman[73158]: 2024-04-12 23:09:34.186324216 +0000 UTC m=+0.433092609 container init 8815b9be6d25536a08a8e71e8797c0ef453367d5e766d3129ff63691604dcb7c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-12T23:09:34.416 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:34 smithi144 podman[73158]: 2024-04-12 23:09:34.192120306 +0000 UTC m=+0.438888708 container start 8815b9be6d25536a08a8e71e8797c0ef453367d5e766d3129ff63691604dcb7c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-12T23:09:34.417 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:34 smithi144 podman[73158]: 2024-04-12 23:09:34.306074835 +0000 UTC m=+0.552843236 container attach 8815b9be6d25536a08a8e71e8797c0ef453367d5e766d3129ff63691604dcb7c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:09:34.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 199214 11.36 MB/sec execute 295 sec latency 44.067 ms 2024-04-12T23:09:34.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:33 smithi181 ceph-mon[52974]: pgmap v63: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 571 op/s 2024-04-12T23:09:34.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:33 smithi181 ceph-mon[52974]: osdmap e78: 8 total, 7 up, 8 in 2024-04-12T23:09:34.668 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:34 smithi144 podman[73158]: 2024-04-12 23:09:34.582555417 +0000 UTC m=+0.829323815 container died 8815b9be6d25536a08a8e71e8797c0ef453367d5e766d3129ff63691604dcb7c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:09:35.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 199566 11.33 MB/sec execute 296 sec latency 39.535 ms 2024-04-12T23:09:35.941 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:35 smithi144 podman[73183]: 2024-04-12 23:09:35.730562645 +0000 UTC m=+1.141844770 container remove 8815b9be6d25536a08a8e71e8797c0ef453367d5e766d3129ff63691604dcb7c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-12T23:09:35.941 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:35 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-12T23:09:35.941 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:35 smithi144 systemd[1]: Stopped Ceph osd.0 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:09:35.941 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:35 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.0.service: Consumed 3min 32.605s CPU time. 2024-04-12T23:09:36.197 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:36 smithi144 systemd[1]: Starting Ceph osd.0 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:09:36.197 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:35 smithi144 ceph-mon[67032]: pgmap v65: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 714 op/s 2024-04-12T23:09:36.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 199776 11.30 MB/sec execute 297 sec latency 52.511 ms 2024-04-12T23:09:36.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:35 smithi181 ceph-mon[52974]: pgmap v65: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 714 op/s 2024-04-12T23:09:36.453 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:36 smithi144 podman[73314]: 2024-04-12 23:09:36.225724998 +0000 UTC m=+0.022243737 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:09:36.453 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:36 smithi144 podman[73314]: 2024-04-12 23:09:36.357151715 +0000 UTC m=+0.153670437 container create 75e2ddfd75d21a7304ef4ab4566acfaf532d5ace76050f3b0368e6ac0b09c2e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T23:09:36.706 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:36 smithi144 podman[73314]: 2024-04-12 23:09:36.610099811 +0000 UTC m=+0.406618551 container init 75e2ddfd75d21a7304ef4ab4566acfaf532d5ace76050f3b0368e6ac0b09c2e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:09:36.706 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:36 smithi144 podman[73314]: 2024-04-12 23:09:36.619869637 +0000 UTC m=+0.416388362 container start 75e2ddfd75d21a7304ef4ab4566acfaf532d5ace76050f3b0368e6ac0b09c2e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate, ceph=True, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:09:36.706 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:36 smithi144 podman[73314]: 2024-04-12 23:09:36.718348004 +0000 UTC m=+0.514866728 container attach 75e2ddfd75d21a7304ef4ab4566acfaf532d5ace76050f3b0368e6ac0b09c2e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240409) 2024-04-12T23:09:37.218 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T23:09:37.218 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T23:09:37.218 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T23:09:37.218 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate[73328]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-12T23:09:37.219 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 bash[73314]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-12T23:09:37.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 200015 11.28 MB/sec execute 298 sec latency 57.995 ms 2024-04-12T23:09:37.476 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:37 smithi144 podman[73314]: 2024-04-12 23:09:37.233344952 +0000 UTC m=+1.029863692 container died 75e2ddfd75d21a7304ef4ab4566acfaf532d5ace76050f3b0368e6ac0b09c2e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T23:09:37.979 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:37 smithi144 ceph-mon[67032]: pgmap v66: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.6 MiB/s wr, 349 op/s 2024-04-12T23:09:38.231 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 podman[73464]: 2024-04-12 23:09:38.242666909 +0000 UTC m=+1.002078464 container remove 75e2ddfd75d21a7304ef4ab4566acfaf532d5ace76050f3b0368e6ac0b09c2e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-activate, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T23:09:38.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 200406 11.29 MB/sec execute 299 sec latency 67.624 ms 2024-04-12T23:09:38.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:37 smithi181 ceph-mon[52974]: pgmap v66: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.6 MiB/s wr, 349 op/s 2024-04-12T23:09:38.486 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 podman[73525]: 2024-04-12 23:09:38.394659316 +0000 UTC m=+0.025290512 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:09:38.726 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 podman[73525]: 2024-04-12 23:09:38.540899434 +0000 UTC m=+0.171530614 container create 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:09:39.009 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 podman[73525]: 2024-04-12 23:09:38.874865974 +0000 UTC m=+0.505497168 container init 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-12T23:09:39.009 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 podman[73525]: 2024-04-12 23:09:38.888589076 +0000 UTC m=+0.519220260 container start 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409) 2024-04-12T23:09:39.009 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 ceph-osd[73543]: -- 172.21.15.144:0/799240863 <== mon.0 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55661f71d680 con 0x55661f6e6c00 2024-04-12T23:09:39.009 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 bash[73525]: 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 2024-04-12T23:09:39.010 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:38 smithi144 systemd[1]: Started Ceph osd.0 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:09:39.260 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:39 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 202/1530 objects degraded (13.203%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:39.348 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:39 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 202/1530 objects degraded (13.203%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:39.418 INFO:teuthology.orchestra.run.smithi144.stdout: 5 200852 11.28 MB/sec execute 300 sec latency 64.203 ms 2024-04-12T23:09:39.511 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:39 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:09:39.463+0000 7fe33cc297c0 -1 Falling back to public interface 2024-04-12T23:09:40.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:40 smithi144 ceph-mon[67032]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 335 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 586 op/s; 202/1530 objects degraded (13.203%) 2024-04-12T23:09:40.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:40.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:40.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:40.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:40 smithi181 ceph-mon[52974]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 335 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 586 op/s; 202/1530 objects degraded (13.203%) 2024-04-12T23:09:40.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:40.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:40.362 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:40.419 INFO:teuthology.orchestra.run.smithi144.stdout: 5 201163 11.27 MB/sec execute 301 sec latency 66.761 ms 2024-04-12T23:09:41.419 INFO:teuthology.orchestra.run.smithi144.stdout: 5 201417 11.25 MB/sec execute 302 sec latency 40.301 ms 2024-04-12T23:09:42.063 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:42 smithi144 ceph-mon[67032]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 335 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.2 MiB/s wr, 211 op/s; 202/1530 objects degraded (13.203%) 2024-04-12T23:09:42.063 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:42.063 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:42.419 INFO:teuthology.orchestra.run.smithi144.stdout: 5 202235 11.32 MB/sec execute 303 sec latency 31.226 ms 2024-04-12T23:09:42.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:42 smithi181 ceph-mon[52974]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 335 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.2 MiB/s wr, 211 op/s; 202/1530 objects degraded (13.203%) 2024-04-12T23:09:42.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:42.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:43.419 INFO:teuthology.orchestra.run.smithi144.stdout: 5 202729 11.34 MB/sec execute 304 sec latency 25.088 ms 2024-04-12T23:09:44.217 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:44 smithi144 ceph-mon[67032]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 454 op/s; 218/1641 objects degraded (13.285%) 2024-04-12T23:09:44.419 INFO:teuthology.orchestra.run.smithi144.stdout: 5 203381 11.32 MB/sec execute 305 sec latency 26.858 ms 2024-04-12T23:09:44.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:44 smithi181 ceph-mon[52974]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 454 op/s; 218/1641 objects degraded (13.285%) 2024-04-12T23:09:45.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 203897 11.35 MB/sec execute 306 sec latency 35.055 ms 2024-04-12T23:09:46.261 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:46 smithi144 ceph-mon[67032]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.7 MiB/s wr, 380 op/s; 218/1641 objects degraded (13.285%) 2024-04-12T23:09:46.261 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:46 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 218/1641 objects degraded (13.285%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:46.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 204362 11.32 MB/sec execute 307 sec latency 46.078 ms 2024-04-12T23:09:46.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:46 smithi181 ceph-mon[52974]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.7 MiB/s wr, 380 op/s; 218/1641 objects degraded (13.285%) 2024-04-12T23:09:46.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:46 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 218/1641 objects degraded (13.285%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:47.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 204706 11.32 MB/sec execute 308 sec latency 61.985 ms 2024-04-12T23:09:48.038 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:47 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:09:47.813+0000 7fe33cc297c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-12T23:09:48.289 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:48 smithi144 ceph-mon[67032]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 378 op/s; 218/1641 objects degraded (13.285%) 2024-04-12T23:09:48.289 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:48.289 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:48.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 205029 11.31 MB/sec execute 309 sec latency 95.525 ms 2024-04-12T23:09:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:48 smithi181 ceph-mon[52974]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 339 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 378 op/s; 218/1641 objects degraded (13.285%) 2024-04-12T23:09:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:49.297 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:49 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:09:49.302+0000 7fe33cc297c0 -1 osd.0 76 log_to_monitors true 2024-04-12T23:09:49.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 205478 11.33 MB/sec execute 310 sec latency 34.780 ms 2024-04-12T23:09:50.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:50 smithi144 ceph-mon[67032]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 713 op/s; 205/1590 objects degraded (12.893%) 2024-04-12T23:09:50.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:50 smithi144 ceph-mon[67032]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T23:09:50.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:50.358 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:50.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 205884 11.35 MB/sec execute 311 sec latency 105.648 ms 2024-04-12T23:09:50.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:50 smithi181 ceph-mon[52974]: pgmap v72: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 713 op/s; 205/1590 objects degraded (12.893%) 2024-04-12T23:09:50.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:50 smithi181 ceph-mon[52974]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-12T23:09:50.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:50.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:50.609 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:09:50 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:09:50.601+0000 7fe32a079700 -1 osd.0 76 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:09:51.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:51 smithi144 ceph-mon[67032]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T23:09:51.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:51 smithi144 ceph-mon[67032]: osdmap e79: 8 total, 7 up, 8 in 2024-04-12T23:09:51.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:51 smithi144 ceph-mon[67032]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:09:51.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:51 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 205/1590 objects degraded (12.893%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:51.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:51 smithi144 ceph-mon[67032]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' 2024-04-12T23:09:51.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 206306 11.35 MB/sec execute 312 sec latency 128.649 ms 2024-04-12T23:09:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:51 smithi181 ceph-mon[52974]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-12T23:09:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:51 smithi181 ceph-mon[52974]: osdmap e79: 8 total, 7 up, 8 in 2024-04-12T23:09:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:51 smithi181 ceph-mon[52974]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:09:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:51 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 205/1590 objects degraded (12.893%), 29 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:51.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:51 smithi181 ceph-mon[52974]: from='osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202]' entity='osd.0' 2024-04-12T23:09:52.222 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:52 smithi144 ceph-mon[67032]: pgmap v74: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 667 op/s; 205/1590 objects degraded (12.893%) 2024-04-12T23:09:52.223 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:52 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:09:52.223 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:52 smithi144 ceph-mon[67032]: osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202] boot 2024-04-12T23:09:52.223 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:52 smithi144 ceph-mon[67032]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T23:09:52.223 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:09:52.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 206385 11.32 MB/sec execute 313 sec latency 827.690 ms 2024-04-12T23:09:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:52 smithi181 ceph-mon[52974]: pgmap v74: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 667 op/s; 205/1590 objects degraded (12.893%) 2024-04-12T23:09:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:52 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:09:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:52 smithi181 ceph-mon[52974]: osd.0 [v2:172.21.15.144:6802/2620095202,v1:172.21.15.144:6803/2620095202] boot 2024-04-12T23:09:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:52 smithi181 ceph-mon[52974]: osdmap e80: 8 total, 8 up, 8 in 2024-04-12T23:09:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-12T23:09:53.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 206385 11.28 MB/sec execute 314 sec latency 1827.961 ms 2024-04-12T23:09:53.891 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:53 smithi144 ceph-mon[67032]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T23:09:53.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:53 smithi181 ceph-mon[52974]: osdmap e81: 8 total, 8 up, 8 in 2024-04-12T23:09:54.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 206579 11.25 MB/sec execute 315 sec latency 2334.431 ms 2024-04-12T23:09:54.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:54 smithi144 ceph-mon[67032]: pgmap v77: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.08k op/s; 88/1737 objects degraded (5.066%) 2024-04-12T23:09:54.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:54 smithi144 ceph-mon[67032]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T23:09:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:54 smithi181 ceph-mon[52974]: pgmap v77: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 1.08k op/s; 88/1737 objects degraded (5.066%) 2024-04-12T23:09:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:54 smithi181 ceph-mon[52974]: osdmap e82: 8 total, 8 up, 8 in 2024-04-12T23:09:55.419 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:09:55.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 206663 11.22 MB/sec execute 316 sec latency 808.039 ms 2024-04-12T23:09:55.889 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T23:09:55.889 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.889 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.889 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 88/1737 objects degraded (5.066%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:55.890 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:09:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: osdmap e83: 8 total, 8 up, 8 in 2024-04-12T23:09:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 88/1737 objects degraded (5.066%), 8 pgs degraded (PG_DEGRADED) 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:09:56.364 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:09:56.364 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (4m) 8s ago 13m 16.1M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:09:56.364 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (56s) 8s ago 13m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:09:56.364 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (49s) 44s ago 12m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 8s ago 13m - - 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (7m) 8s ago 7m 38.3M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (7m) 44s ago 7m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (3m) 8s ago 14m 503M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (2m) 44s ago 12m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (2m) 8s ago 15m 46.9M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (114s) 44s ago 12m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (7m) 8s ago 7m 276M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (7m) 44s ago 7m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (4m) 8s ago 13m 8954k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (3m) 44s ago 12m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (17s) 8s ago 12m 11.6M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (11m) 8s ago 11m 339M 1936M 17.2.0 e1d6a67b021e 5a74f7ccb430 2024-04-12T23:09:56.365 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (11m) 8s ago 11m 520M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:09:56.366 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (10m) 8s ago 10m 350M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:09:56.366 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (10m) 44s ago 10m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:09:56.366 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (10m) 44s ago 10m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:09:56.366 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (9m) 44s ago 9m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:09:56.366 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (9m) 44s ago 9m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:09:56.366 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (2m) 8s ago 12m 60.4M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:09:56.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 206886 11.19 MB/sec execute 317 sec latency 1032.420 ms 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: pgmap v80: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 6.5 MiB/s wr, 610 op/s; 88/1737 objects degraded (5.066%) 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: pgmap v81: 97 pgs: 24 peering, 4 active+undersized+degraded, 69 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 7.4 MiB/s wr, 677 op/s; 45/1740 objects degraded (2.586%); 0 B/s, 14 objects/s recovering 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: pgmap v82: 97 pgs: 24 peering, 4 active+undersized+degraded, 69 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 417 KiB/s wr, 19 op/s; 45/1740 objects degraded (2.586%); 0 B/s, 20 objects/s recovering 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:56.648 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T23:09:56.649 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:56.649 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: pgmap v80: 97 pgs: 23 peering, 8 active+undersized+degraded, 5 active+undersized, 61 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.7 KiB/s rd, 6.5 MiB/s wr, 610 op/s; 88/1737 objects degraded (5.066%) 2024-04-12T23:09:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: pgmap v81: 97 pgs: 24 peering, 4 active+undersized+degraded, 69 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.1 KiB/s rd, 7.4 MiB/s wr, 677 op/s; 45/1740 objects degraded (2.586%); 0 B/s, 14 objects/s recovering 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: pgmap v82: 97 pgs: 24 peering, 4 active+undersized+degraded, 69 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 417 KiB/s wr, 19 op/s; 45/1740 objects degraded (2.586%); 0 B/s, 20 objects/s recovering 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: osdmap e84: 8 total, 8 up, 8 in 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:09:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:09:56.958 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:09:56.958 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:09:56.958 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:09:56.958 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:56.958 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 5 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:09:56.959 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:09:57.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 207237 11.19 MB/sec execute 318 sec latency 751.643 ms 2024-04-12T23:09:57.470 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:09:57.470 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:09:57.470 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:09:57.470 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:09:57.471 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:09:57.471 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:09:57.471 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:09:57.471 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:09:57.471 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:09:57.471 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "7/23 daemons upgraded", 2024-04-12T23:09:57.472 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:09:57.472 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:09:57.472 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:09:57.681 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:57 smithi144 ceph-mon[67032]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:57.681 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:57 smithi144 ceph-mon[67032]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:57.681 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:57 smithi144 ceph-mon[67032]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T23:09:57.681 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:57 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/476852915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:57 smithi181 ceph-mon[52974]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:57 smithi181 ceph-mon[52974]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:57 smithi181 ceph-mon[52974]: osdmap e85: 8 total, 8 up, 8 in 2024-04-12T23:09:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:57 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/476852915' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-12T23:09:58.116 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.f is active+recovering+undersized+degraded+remapped, acting [7,4] 2024-04-12T23:09:58.117 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-12T23:09:58.117 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.17 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-12T23:09:58.117 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-04-12T23:09:58.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 207614 11.19 MB/sec execute 319 sec latency 1006.476 ms 2024-04-12T23:09:58.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:58 smithi144 ceph-mon[67032]: pgmap v85: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 5.0 MiB/s wr, 483 op/s; 160/1491 objects degraded (10.731%); 3.2 MiB/s, 52 objects/s recovering 2024-04-12T23:09:58.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:58 smithi144 ceph-mon[67032]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:58.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:58 smithi144 ceph-mon[67032]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T23:09:58.701 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:58 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3682553354' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:09:58.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:58 smithi181 ceph-mon[52974]: pgmap v85: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 15 KiB/s rd, 5.0 MiB/s wr, 483 op/s; 160/1491 objects degraded (10.731%); 3.2 MiB/s, 52 objects/s recovering 2024-04-12T23:09:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:58 smithi181 ceph-mon[52974]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:09:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:58 smithi181 ceph-mon[52974]: osdmap e86: 8 total, 8 up, 8 in 2024-04-12T23:09:58.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:58 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3682553354' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:09:59.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 208007 11.19 MB/sec execute 320 sec latency 917.637 ms 2024-04-12T23:09:59.722 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:09:59 smithi144 ceph-mon[67032]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T23:09:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:09:59 smithi181 ceph-mon[52974]: osdmap e87: 8 total, 8 up, 8 in 2024-04-12T23:10:00.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 208303 11.16 MB/sec execute 321 sec latency 755.326 ms 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pgmap v88: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 3.0 MiB/s wr, 308 op/s; 160/1491 objects degraded (10.731%); 2.1 MiB/s, 21 objects/s recovering 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: [WRN] PG_DEGRADED: Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-04-12T23:10:00.765 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-12T23:10:00.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-12T23:10:00.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.f is active+recovering+undersized+degraded+remapped, acting [7,4] 2024-04-12T23:10:00.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-12T23:10:00.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.17 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-12T23:10:00.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-04-12T23:10:00.766 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:00 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pgmap v88: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 89 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 3.0 MiB/s wr, 308 op/s; 160/1491 objects degraded (10.731%); 2.1 MiB/s, 21 objects/s recovering 2024-04-12T23:10:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: osdmap e88: 8 total, 8 up, 8 in 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: [WRN] PG_DEGRADED: Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.f is active+recovering+undersized+degraded+remapped, acting [7,4] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.17 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-04-12T23:10:00.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:00 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 160/1491 objects degraded (10.731%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:01.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 208573 11.15 MB/sec execute 322 sec latency 1020.818 ms 2024-04-12T23:10:01.790 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:01 smithi144 ceph-mon[67032]: osdmap e89: 8 total, 8 up, 8 in 2024-04-12T23:10:01.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:01 smithi181 ceph-mon[52974]: osdmap e89: 8 total, 8 up, 8 in 2024-04-12T23:10:02.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 208753 11.13 MB/sec execute 323 sec latency 59.473 ms 2024-04-12T23:10:02.829 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:02 smithi144 ceph-mon[67032]: pgmap v91: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s wr, 48 op/s; 62/1521 objects degraded (4.076%); 0 B/s, 24 objects/s recovering 2024-04-12T23:10:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:02 smithi181 ceph-mon[52974]: pgmap v91: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 1.1 MiB/s wr, 48 op/s; 62/1521 objects degraded (4.076%); 0 B/s, 24 objects/s recovering 2024-04-12T23:10:03.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 209040 11.13 MB/sec execute 324 sec latency 50.724 ms 2024-04-12T23:10:03.879 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:03 smithi144 ceph-mon[67032]: pgmap v92: 97 pgs: 3 peering, 94 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 0 B/s, 29 objects/s recovering 2024-04-12T23:10:03.879 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:03 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/1521 objects degraded (4.076%), 3 pgs degraded) 2024-04-12T23:10:04.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:03 smithi181 ceph-mon[52974]: pgmap v92: 97 pgs: 3 peering, 94 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 0 B/s, 29 objects/s recovering 2024-04-12T23:10:04.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:03 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 62/1521 objects degraded (4.076%), 3 pgs degraded) 2024-04-12T23:10:04.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 209286 11.13 MB/sec execute 325 sec latency 71.774 ms 2024-04-12T23:10:05.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 209652 11.12 MB/sec execute 326 sec latency 47.779 ms 2024-04-12T23:10:06.420 INFO:teuthology.orchestra.run.smithi144.stdout: 5 209946 11.12 MB/sec execute 327 sec latency 49.453 ms 2024-04-12T23:10:06.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:06 smithi144 ceph-mon[67032]: pgmap v93: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 374 op/s; 0 B/s, 27 objects/s recovering 2024-04-12T23:10:06.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:06 smithi181 ceph-mon[52974]: pgmap v93: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 374 op/s; 0 B/s, 27 objects/s recovering 2024-04-12T23:10:07.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 210121 11.10 MB/sec execute 328 sec latency 54.380 ms 2024-04-12T23:10:08.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 210350 11.09 MB/sec execute 329 sec latency 75.085 ms 2024-04-12T23:10:08.529 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:08 smithi144 ceph-mon[67032]: pgmap v94: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 538 op/s; 0 B/s, 22 objects/s recovering 2024-04-12T23:10:08.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:08 smithi181 ceph-mon[52974]: pgmap v94: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 538 op/s; 0 B/s, 22 objects/s recovering 2024-04-12T23:10:09.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 210686 11.07 MB/sec execute 330 sec latency 74.795 ms 2024-04-12T23:10:10.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 211014 11.07 MB/sec execute 331 sec latency 36.779 ms 2024-04-12T23:10:10.574 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:10 smithi144 ceph-mon[67032]: pgmap v95: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 0 B/s, 19 objects/s recovering 2024-04-12T23:10:10.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:10 smithi181 ceph-mon[52974]: pgmap v95: 97 pgs: 97 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s; 0 B/s, 19 objects/s recovering 2024-04-12T23:10:11.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 211345 11.06 MB/sec execute 332 sec latency 53.369 ms 2024-04-12T23:10:11.594 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:10:11.594 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:11 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:10:11.594 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:11 smithi144 ceph-mon[67032]: Upgrade: osd.1 is safe to restart 2024-04-12T23:10:11.594 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:11.594 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:11.594 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:11.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:10:11.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:11 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-12T23:10:11.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:11 smithi181 ceph-mon[52974]: Upgrade: osd.1 is safe to restart 2024-04-12T23:10:11.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:11.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:11.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:12.361 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:12 smithi144 ceph-mon[67032]: pgmap v96: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 396 op/s; 0 B/s, 7 objects/s recovering 2024-04-12T23:10:12.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 211635 11.04 MB/sec execute 333 sec latency 57.273 ms 2024-04-12T23:10:12.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:12 smithi181 ceph-mon[52974]: pgmap v96: 97 pgs: 97 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 396 op/s; 0 B/s, 7 objects/s recovering 2024-04-12T23:10:13.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 212045 11.03 MB/sec execute 334 sec latency 37.492 ms 2024-04-12T23:10:14.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 212245 11.01 MB/sec execute 335 sec latency 66.818 ms 2024-04-12T23:10:14.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:14 smithi144 ceph-mon[67032]: pgmap v97: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 534 op/s; 0 B/s, 7 objects/s recovering 2024-04-12T23:10:14.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:14 smithi144 ceph-mon[67032]: Upgrade: Updating osd.1 2024-04-12T23:10:14.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:14.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T23:10:14.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:10:14.690 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:14 smithi144 ceph-mon[67032]: Deploying daemon osd.1 on smithi144 2024-04-12T23:10:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:14 smithi181 ceph-mon[52974]: pgmap v97: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 534 op/s; 0 B/s, 7 objects/s recovering 2024-04-12T23:10:14.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:14 smithi181 ceph-mon[52974]: Upgrade: Updating osd.1 2024-04-12T23:10:14.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:14.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-12T23:10:14.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:10:14.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:14 smithi181 ceph-mon[52974]: Deploying daemon osd.1 on smithi144 2024-04-12T23:10:15.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 212584 11.02 MB/sec execute 336 sec latency 44.937 ms 2024-04-12T23:10:16.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 213026 11.03 MB/sec execute 337 sec latency 39.612 ms 2024-04-12T23:10:16.763 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:16 smithi144 ceph-mon[67032]: pgmap v98: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 0 B/s, 1 objects/s recovering 2024-04-12T23:10:16.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:16 smithi181 ceph-mon[52974]: pgmap v98: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 0 B/s, 1 objects/s recovering 2024-04-12T23:10:17.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 213399 11.04 MB/sec execute 338 sec latency 44.009 ms 2024-04-12T23:10:18.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 213663 11.02 MB/sec execute 339 sec latency 70.551 ms 2024-04-12T23:10:18.794 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:18 smithi144 ceph-mon[67032]: pgmap v99: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 539 op/s 2024-04-12T23:10:18.795 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:18 smithi144 systemd[1]: Stopping Ceph osd.1 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:10:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:18 smithi181 ceph-mon[52974]: pgmap v99: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 539 op/s 2024-04-12T23:10:19.048 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[39332]: 2024-04-12T23:10:18.957+0000 7f48f0c2e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:10:19.048 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[39332]: 2024-04-12T23:10:18.957+0000 7f48f0c2e700 -1 osd.1 89 *** Got signal Terminated *** 2024-04-12T23:10:19.048 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[39332]: 2024-04-12T23:10:18.957+0000 7f48f0c2e700 -1 osd.1 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:10:19.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 214013 11.01 MB/sec execute 340 sec latency 38.233 ms 2024-04-12T23:10:19.836 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:19 smithi144 ceph-mon[67032]: osd.1 marked itself down and dead 2024-04-12T23:10:19.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:19 smithi181 ceph-mon[52974]: osd.1 marked itself down and dead 2024-04-12T23:10:20.347 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:20 smithi144 podman[75650]: 2024-04-12 23:10:20.258318218 +0000 UTC m=+1.482869850 container died 5a74f7ccb430325b6e826766ae1a5cee0ab79234a175549c26f5b863f0ea27a3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, architecture=x86_64, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-12T23:10:20.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 214297 11.00 MB/sec execute 341 sec latency 711.269 ms 2024-04-12T23:10:20.636 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:20 smithi144 ceph-mon[67032]: pgmap v100: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-04-12T23:10:20.636 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:20 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:10:20.636 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:20 smithi144 ceph-mon[67032]: osdmap e90: 8 total, 7 up, 8 in 2024-04-12T23:10:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:20 smithi181 ceph-mon[52974]: pgmap v100: 97 pgs: 97 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-04-12T23:10:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:20 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:10:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:20 smithi181 ceph-mon[52974]: osdmap e90: 8 total, 7 up, 8 in 2024-04-12T23:10:21.155 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:20 smithi144 podman[75650]: 2024-04-12 23:10:20.968127138 +0000 UTC m=+2.192678766 container cleanup 5a74f7ccb430325b6e826766ae1a5cee0ab79234a175549c26f5b863f0ea27a3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-type=git, maintainer=Guillaume Abrioux ) 2024-04-12T23:10:21.155 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:20 smithi144 bash[75650]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1 2024-04-12T23:10:21.407 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:21 smithi144 podman[75670]: 2024-04-12 23:10:21.353685173 +0000 UTC m=+1.093984344 container remove 5a74f7ccb430325b6e826766ae1a5cee0ab79234a175549c26f5b863f0ea27a3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8, distribution-scope=public, name=centos-stream, RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T23:10:21.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 214555 10.99 MB/sec execute 342 sec latency 924.399 ms 2024-04-12T23:10:21.658 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:21 smithi144 ceph-mon[67032]: osdmap e91: 8 total, 7 up, 8 in 2024-04-12T23:10:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:21 smithi181 ceph-mon[52974]: osdmap e91: 8 total, 7 up, 8 in 2024-04-12T23:10:22.161 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:21 smithi144 podman[75757]: 2024-04-12 23:10:21.855989889 +0000 UTC m=+0.026513870 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:10:22.161 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:21 smithi144 podman[75757]: 2024-04-12 23:10:21.988335337 +0000 UTC m=+0.158859315 container create c71c625c965182e586bb32ef5a5460b67625440079436e848c8b7093a5c5fb5d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb) 2024-04-12T23:10:22.412 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:22 smithi144 podman[75757]: 2024-04-12 23:10:22.402514225 +0000 UTC m=+0.573038207 container init c71c625c965182e586bb32ef5a5460b67625440079436e848c8b7093a5c5fb5d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-12T23:10:22.412 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:22 smithi144 podman[75757]: 2024-04-12 23:10:22.409806567 +0000 UTC m=+0.580330549 container start c71c625c965182e586bb32ef5a5460b67625440079436e848c8b7093a5c5fb5d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T23:10:22.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 215014 10.99 MB/sec execute 343 sec latency 44.361 ms 2024-04-12T23:10:22.664 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:22 smithi144 ceph-mon[67032]: pgmap v103: 97 pgs: 3 peering, 13 stale+active+clean, 81 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.7 MiB/s wr, 312 op/s 2024-04-12T23:10:22.664 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:22 smithi144 podman[75757]: 2024-04-12 23:10:22.604508272 +0000 UTC m=+0.775032272 container attach c71c625c965182e586bb32ef5a5460b67625440079436e848c8b7093a5c5fb5d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-04-12T23:10:22.916 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:22 smithi144 podman[75757]: 2024-04-12 23:10:22.877960421 +0000 UTC m=+1.048484402 container died c71c625c965182e586bb32ef5a5460b67625440079436e848c8b7093a5c5fb5d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-12T23:10:22.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:22 smithi181 ceph-mon[52974]: pgmap v103: 97 pgs: 3 peering, 13 stale+active+clean, 81 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.7 MiB/s wr, 312 op/s 2024-04-12T23:10:23.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 215576 10.98 MB/sec execute 344 sec latency 36.379 ms 2024-04-12T23:10:23.671 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:23 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 148/1623 objects degraded (9.119%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:23.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:23 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 148/1623 objects degraded (9.119%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:24.178 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:23 smithi144 podman[75785]: 2024-04-12 23:10:23.961385874 +0000 UTC m=+1.076833330 container remove c71c625c965182e586bb32ef5a5460b67625440079436e848c8b7093a5c5fb5d (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:10:24.178 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:23 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-12T23:10:24.179 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:23 smithi144 systemd[1]: Stopped Ceph osd.1 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:10:24.179 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:23 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.1.service: Consumed 3min 53.246s CPU time. 2024-04-12T23:10:24.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 216039 10.99 MB/sec execute 345 sec latency 28.055 ms 2024-04-12T23:10:24.432 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:24 smithi144 systemd[1]: Starting Ceph osd.1 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:10:24.683 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:24 smithi144 ceph-mon[67032]: pgmap v104: 97 pgs: 21 active+undersized+degraded, 3 peering, 15 active+undersized, 58 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.7 MiB/s wr, 578 op/s; 148/1623 objects degraded (9.119%) 2024-04-12T23:10:24.683 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:24 smithi144 podman[75877]: 2024-04-12 23:10:24.567133495 +0000 UTC m=+0.024369120 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:10:24.683 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:24 smithi144 podman[75877]: 2024-04-12 23:10:24.686170365 +0000 UTC m=+0.143406023 container create c64d79faf862ac0b883f9cc743ec482ee7f5febe4f9007d59956db90aa8afea9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-12T23:10:24.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:24 smithi181 ceph-mon[52974]: pgmap v104: 97 pgs: 21 active+undersized+degraded, 3 peering, 15 active+undersized, 58 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.7 MiB/s wr, 578 op/s; 148/1623 objects degraded (9.119%) 2024-04-12T23:10:25.195 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 podman[75877]: 2024-04-12 23:10:25.008497547 +0000 UTC m=+0.465733164 container init c64d79faf862ac0b883f9cc743ec482ee7f5febe4f9007d59956db90aa8afea9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_CLEAN=True, ceph=True) 2024-04-12T23:10:25.195 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 podman[75877]: 2024-04-12 23:10:25.014230525 +0000 UTC m=+0.471466138 container start c64d79faf862ac0b883f9cc743ec482ee7f5febe4f9007d59956db90aa8afea9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb) 2024-04-12T23:10:25.195 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 podman[75877]: 2024-04-12 23:10:25.183396171 +0000 UTC m=+0.640631787 container attach c64d79faf862ac0b883f9cc743ec482ee7f5febe4f9007d59956db90aa8afea9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-12T23:10:25.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 216532 11.02 MB/sec execute 346 sec latency 34.037 ms 2024-04-12T23:10:25.706 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 bash[75877]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-12T23:10:25.707 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate[75891]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-12T23:10:25.708 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:25 smithi144 podman[75877]: 2024-04-12 23:10:25.697425291 +0000 UTC m=+1.154660915 container died c64d79faf862ac0b883f9cc743ec482ee7f5febe4f9007d59956db90aa8afea9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:10:26.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 216978 11.03 MB/sec execute 347 sec latency 38.425 ms 2024-04-12T23:10:26.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:26 smithi144 ceph-mon[67032]: pgmap v105: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 482 op/s; 163/1596 objects degraded (10.213%) 2024-04-12T23:10:26.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:26.714 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:26 smithi181 ceph-mon[52974]: pgmap v105: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 482 op/s; 163/1596 objects degraded (10.213%) 2024-04-12T23:10:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:27.421 INFO:teuthology.orchestra.run.smithi144.stdout: 5 217274 11.01 MB/sec execute 348 sec latency 34.022 ms 2024-04-12T23:10:27.476 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:27 smithi144 podman[76018]: 2024-04-12 23:10:27.33433543 +0000 UTC m=+1.627963755 container remove c64d79faf862ac0b883f9cc743ec482ee7f5febe4f9007d59956db90aa8afea9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T23:10:27.733 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:27 smithi144 podman[76072]: 2024-04-12 23:10:27.503983666 +0000 UTC m=+0.030598444 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:10:27.733 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:27 smithi144 podman[76072]: 2024-04-12 23:10:27.702740808 +0000 UTC m=+0.229355556 container create 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True) 2024-04-12T23:10:28.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 217581 11.00 MB/sec execute 349 sec latency 40.800 ms 2024-04-12T23:10:28.492 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:28 smithi144 podman[76072]: 2024-04-12 23:10:28.375593708 +0000 UTC m=+0.902208459 container init 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T23:10:28.492 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:28 smithi144 podman[76072]: 2024-04-12 23:10:28.382597066 +0000 UTC m=+0.909211812 container start 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T23:10:28.492 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:28 smithi144 ceph-osd[76113]: -- 172.21.15.144:0/3034523147 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c4f7067680 con 0x55c4f7030c00 2024-04-12T23:10:28.617 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:10:28.746 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:28 smithi144 ceph-mon[67032]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 716 op/s; 178/1677 objects degraded (10.614%) 2024-04-12T23:10:28.746 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:28.746 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:28.746 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:10:28.746 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:28 smithi144 bash[76072]: 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 2024-04-12T23:10:28.747 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:28 smithi144 systemd[1]: Started Ceph osd.1 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:10:28.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:28 smithi181 ceph-mon[52974]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 716 op/s; 178/1677 objects degraded (10.614%) 2024-04-12T23:10:28.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:28.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:28.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:10:29.016 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:28 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:10:28.956+0000 7f67518387c0 -1 Falling back to public interface 2024-04-12T23:10:29.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 217865 10.99 MB/sec execute 350 sec latency 70.363 ms 2024-04-12T23:10:29.654 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:10:29.654 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (5m) 41s ago 14m 16.1M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:10:29.654 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (89s) 41s ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:10:29.654 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (82s) 78s ago 13m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:10:29.654 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 41s ago 13m - - 2024-04-12T23:10:29.654 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (8m) 41s ago 8m 38.3M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (8m) 78s ago 8m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (4m) 41s ago 15m 503M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (3m) 78s ago 13m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (2m) 41s ago 15m 46.9M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (2m) 78s ago 13m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (8m) 41s ago 8m 276M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (7m) 78s ago 7m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (4m) 41s ago 13m 8954k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (4m) 78s ago 13m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (51s) 41s ago 12m 11.6M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 starting - - - 1936M 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (11m) 41s ago 11m 520M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (11m) 41s ago 11m 350M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (11m) 78s ago 11m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:10:29.655 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (10m) 78s ago 10m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:10:29.656 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (10m) 78s ago 10m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:10:29.656 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (9m) 78s ago 9m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:10:29.656 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (3m) 41s ago 12m 60.4M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:10:29.777 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:29 smithi144 ceph-mon[67032]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:29.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:29 smithi181 ceph-mon[52974]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:10:30.217 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 5 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:10:30.218 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:10:30.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 218214 10.98 MB/sec execute 351 sec latency 80.400 ms 2024-04-12T23:10:30.760 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "7/23 daemons upgraded", 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:10:30.761 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:10:30.786 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:30 smithi144 ceph-mon[67032]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:30.786 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:30 smithi144 ceph-mon[67032]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 591 op/s; 178/1677 objects degraded (10.614%) 2024-04-12T23:10:30.786 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:30 smithi144 ceph-mon[67032]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:30.786 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:30 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1113114994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:30.786 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:30 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 178/1677 objects degraded (10.614%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:30 smithi181 ceph-mon[52974]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:30 smithi181 ceph-mon[52974]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 591 op/s; 178/1677 objects degraded (10.614%) 2024-04-12T23:10:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:30 smithi181 ceph-mon[52974]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:30 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1113114994' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:30.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:30 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 178/1677 objects degraded (10.614%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:31.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 218791 10.99 MB/sec execute 352 sec latency 27.934 ms 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 178/1677 objects degraded (10.614%), 23 pgs degraded 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout: osd.1 (root=default,host=smithi144) is down 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 178/1677 objects degraded (10.614%), 23 pgs degraded 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-04-12T23:10:31.441 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-04-12T23:10:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-04-12T23:10:31.443 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-04-12T23:10:31.443 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-04-12T23:10:31.443 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-04-12T23:10:31.443 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-04-12T23:10:31.443 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-04-12T23:10:31.794 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:31 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/938051313' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:10:31.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:31 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/938051313' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:10:32.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 219360 10.99 MB/sec execute 353 sec latency 27.798 ms 2024-04-12T23:10:32.806 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:32 smithi144 ceph-mon[67032]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:32.806 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:32 smithi144 ceph-mon[67032]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 541 op/s; 178/1677 objects degraded (10.614%) 2024-04-12T23:10:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:32 smithi181 ceph-mon[52974]: from='client.44207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:10:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:32 smithi181 ceph-mon[52974]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 343 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 541 op/s; 178/1677 objects degraded (10.614%) 2024-04-12T23:10:33.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 219799 11.01 MB/sec execute 354 sec latency 47.858 ms 2024-04-12T23:10:33.829 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:33 smithi144 ceph-mon[67032]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 709 op/s; 183/1662 objects degraded (11.011%) 2024-04-12T23:10:34.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:33 smithi181 ceph-mon[52974]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 709 op/s; 183/1662 objects degraded (11.011%) 2024-04-12T23:10:34.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 220219 11.02 MB/sec execute 355 sec latency 39.416 ms 2024-04-12T23:10:35.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 220602 11.02 MB/sec execute 356 sec latency 49.281 ms 2024-04-12T23:10:36.364 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:36 smithi144 ceph-mon[67032]: pgmap v110: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 532 op/s; 184/1671 objects degraded (11.011%) 2024-04-12T23:10:36.846 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:36 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 184/1671 objects degraded (11.011%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:36.847 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:36 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:10:36.209+0000 7f67518387c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-12T23:10:36.847 INFO:teuthology.orchestra.run.smithi144.stdout: 5 220828 11.01 MB/sec execute 357 sec latency 36.500 ms 2024-04-12T23:10:36.847 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:36 smithi181 ceph-mon[52974]: pgmap v110: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 340 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 532 op/s; 184/1671 objects degraded (11.011%) 2024-04-12T23:10:36.847 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:36 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 184/1671 objects degraded (11.011%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:37.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 221104 10.99 MB/sec execute 358 sec latency 66.770 ms 2024-04-12T23:10:37.634 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:37 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:10:37.468+0000 7f67518387c0 -1 osd.1 89 log_to_monitors true 2024-04-12T23:10:38.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:38 smithi144 ceph-mon[67032]: pgmap v111: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 346 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.9 MiB/s wr, 631 op/s; 189/1680 objects degraded (11.250%) 2024-04-12T23:10:38.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:38.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:38.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:38 smithi144 ceph-mon[67032]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T23:10:38.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 221440 10.98 MB/sec execute 359 sec latency 55.179 ms 2024-04-12T23:10:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:38 smithi181 ceph-mon[52974]: pgmap v111: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 346 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.9 MiB/s wr, 631 op/s; 189/1680 objects degraded (11.250%) 2024-04-12T23:10:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:38 smithi181 ceph-mon[52974]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-12T23:10:38.971 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:10:38 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:10:38.888+0000 7f673ec88700 -1 osd.1 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:10:39.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 221660 10.97 MB/sec execute 360 sec latency 197.280 ms 2024-04-12T23:10:39.478 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:39 smithi144 ceph-mon[67032]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T23:10:39.478 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:39 smithi144 ceph-mon[67032]: osdmap e92: 8 total, 7 up, 8 in 2024-04-12T23:10:39.478 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:39 smithi144 ceph-mon[67032]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:10:39.478 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:39 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:39.478 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:39 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:39.478 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:39 smithi144 ceph-mon[67032]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' 2024-04-12T23:10:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:39 smithi181 ceph-mon[52974]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-12T23:10:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:39 smithi181 ceph-mon[52974]: osdmap e92: 8 total, 7 up, 8 in 2024-04-12T23:10:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:39 smithi181 ceph-mon[52974]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:10:39.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:39 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:39.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:39 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:39.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:39 smithi181 ceph-mon[52974]: from='osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631]' entity='osd.1' 2024-04-12T23:10:40.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 221889 10.95 MB/sec execute 361 sec latency 503.674 ms 2024-04-12T23:10:40.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:40 smithi144 ceph-mon[67032]: pgmap v113: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 189/1680 objects degraded (11.250%) 2024-04-12T23:10:40.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:40 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:10:40.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:40 smithi144 ceph-mon[67032]: osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631] boot 2024-04-12T23:10:40.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:40 smithi144 ceph-mon[67032]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T23:10:40.566 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:40 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:10:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:40 smithi181 ceph-mon[52974]: pgmap v113: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 189/1680 objects degraded (11.250%) 2024-04-12T23:10:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:40 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:10:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:40 smithi181 ceph-mon[52974]: osd.1 [v2:172.21.15.144:6810/4145513631,v1:172.21.15.144:6811/4145513631] boot 2024-04-12T23:10:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:40 smithi181 ceph-mon[52974]: osdmap e93: 8 total, 8 up, 8 in 2024-04-12T23:10:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:40 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-12T23:10:41.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 221918 10.93 MB/sec execute 362 sec latency 1441.432 ms 2024-04-12T23:10:41.487 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:41 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 189/1680 objects degraded (11.250%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:41.487 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:41 smithi144 ceph-mon[67032]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T23:10:41.487 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:41.487 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:41.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:41 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 189/1680 objects degraded (11.250%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:41.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:41 smithi181 ceph-mon[52974]: osdmap e94: 8 total, 8 up, 8 in 2024-04-12T23:10:41.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:41.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:42.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 222256 10.92 MB/sec execute 363 sec latency 1583.005 ms 2024-04-12T23:10:42.499 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:42 smithi144 ceph-mon[67032]: pgmap v116: 97 pgs: 2 peering, 22 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 454 op/s; 177/1674 objects degraded (10.573%) 2024-04-12T23:10:42.499 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:42 smithi144 ceph-mon[67032]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T23:10:42.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:42 smithi181 ceph-mon[52974]: pgmap v116: 97 pgs: 2 peering, 22 active+undersized+degraded, 15 active+undersized, 58 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 454 op/s; 177/1674 objects degraded (10.573%) 2024-04-12T23:10:42.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:42 smithi181 ceph-mon[52974]: osdmap e95: 8 total, 8 up, 8 in 2024-04-12T23:10:43.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 222529 10.89 MB/sec execute 364 sec latency 1018.624 ms 2024-04-12T23:10:43.651 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:43 smithi144 ceph-mon[67032]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T23:10:43.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:43 smithi181 ceph-mon[52974]: osdmap e96: 8 total, 8 up, 8 in 2024-04-12T23:10:44.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 222802 10.89 MB/sec execute 365 sec latency 1027.165 ms 2024-04-12T23:10:44.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:44 smithi144 ceph-mon[67032]: pgmap v119: 97 pgs: 74 active+clean, 5 active+undersized, 2 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.1 MiB/s wr, 449 op/s; 180/1497 objects degraded (12.024%); 1024 KiB/s, 18 objects/s recovering 2024-04-12T23:10:44.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:44 smithi144 ceph-mon[67032]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T23:10:44.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:44 smithi181 ceph-mon[52974]: pgmap v119: 97 pgs: 74 active+clean, 5 active+undersized, 2 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.1 MiB/s wr, 449 op/s; 180/1497 objects degraded (12.024%); 1024 KiB/s, 18 objects/s recovering 2024-04-12T23:10:44.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:44 smithi181 ceph-mon[52974]: osdmap e97: 8 total, 8 up, 8 in 2024-04-12T23:10:45.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 223023 10.87 MB/sec execute 366 sec latency 965.705 ms 2024-04-12T23:10:45.494 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T23:10:45.494 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.494 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.494 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: pgmap v122: 97 pgs: 84 active+clean, 3 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.8 MiB/s wr, 499 op/s; 156/1497 objects degraded (10.421%); 1.2 MiB/s, 66 keys/s, 23 objects/s recovering 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: pgmap v123: 97 pgs: 84 active+clean, 3 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 MiB/s wr, 514 op/s; 156/1497 objects degraded (10.421%); 1.2 MiB/s, 68 keys/s, 23 objects/s recovering 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:10:45.495 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:45 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 180/1497 objects degraded (12.024%), 13 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:45.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: osdmap e98: 8 total, 8 up, 8 in 2024-04-12T23:10:45.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: pgmap v122: 97 pgs: 84 active+clean, 3 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.8 MiB/s wr, 499 op/s; 156/1497 objects degraded (10.421%); 1.2 MiB/s, 66 keys/s, 23 objects/s recovering 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: pgmap v123: 97 pgs: 84 active+clean, 3 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 MiB/s wr, 514 op/s; 156/1497 objects degraded (10.421%); 1.2 MiB/s, 68 keys/s, 23 objects/s recovering 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:10:45.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:45 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 180/1497 objects degraded (12.024%), 13 pgs degraded (PG_DEGRADED) 2024-04-12T23:10:46.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 223313 10.87 MB/sec execute 367 sec latency 984.431 ms 2024-04-12T23:10:46.545 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:46 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:10:46.545 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:46 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-12T23:10:46.545 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:46 smithi144 ceph-mon[67032]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T23:10:46.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:46 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:10:46.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:46 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-12T23:10:46.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:46 smithi181 ceph-mon[52974]: osdmap e99: 8 total, 8 up, 8 in 2024-04-12T23:10:47.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 223555 10.86 MB/sec execute 368 sec latency 42.713 ms 2024-04-12T23:10:47.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:47 smithi144 ceph-mon[67032]: pgmap v125: 97 pgs: 86 active+clean, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 396 KiB/s wr, 36 op/s; 156/1470 objects degraded (10.612%); 391 KiB/s, 66 keys/s, 10 objects/s recovering 2024-04-12T23:10:47.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:47 smithi181 ceph-mon[52974]: pgmap v125: 97 pgs: 86 active+clean, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 334 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 396 KiB/s wr, 36 op/s; 156/1470 objects degraded (10.612%); 391 KiB/s, 66 keys/s, 10 objects/s recovering 2024-04-12T23:10:48.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 223794 10.86 MB/sec execute 369 sec latency 75.990 ms 2024-04-12T23:10:48.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:48 smithi144 ceph-mon[67032]: pgmap v126: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 367 op/s; 1.0 MiB/s, 39 objects/s recovering 2024-04-12T23:10:48.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:48 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 156/1470 objects degraded (10.612%), 7 pgs degraded) 2024-04-12T23:10:49.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:48 smithi181 ceph-mon[52974]: pgmap v126: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 367 op/s; 1.0 MiB/s, 39 objects/s recovering 2024-04-12T23:10:49.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:48 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 156/1470 objects degraded (10.612%), 7 pgs degraded) 2024-04-12T23:10:49.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 224165 10.86 MB/sec execute 370 sec latency 59.662 ms 2024-04-12T23:10:50.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 224648 10.87 MB/sec execute 371 sec latency 42.697 ms 2024-04-12T23:10:51.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 225157 10.87 MB/sec execute 372 sec latency 29.609 ms 2024-04-12T23:10:51.857 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:51 smithi144 ceph-mon[67032]: pgmap v127: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 332 op/s; 869 KiB/s, 33 objects/s recovering 2024-04-12T23:10:51.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:51 smithi181 ceph-mon[52974]: pgmap v127: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 332 op/s; 869 KiB/s, 33 objects/s recovering 2024-04-12T23:10:52.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 225716 10.87 MB/sec execute 373 sec latency 32.641 ms 2024-04-12T23:10:53.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 226287 10.88 MB/sec execute 374 sec latency 45.764 ms 2024-04-12T23:10:53.870 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:53 smithi144 ceph-mon[67032]: pgmap v128: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 469 op/s; 674 KiB/s, 25 objects/s recovering 2024-04-12T23:10:53.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:53 smithi181 ceph-mon[52974]: pgmap v128: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 469 op/s; 674 KiB/s, 25 objects/s recovering 2024-04-12T23:10:54.422 INFO:teuthology.orchestra.run.smithi144.stdout: 5 226784 10.89 MB/sec execute 375 sec latency 41.183 ms 2024-04-12T23:10:55.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 227129 10.88 MB/sec execute 376 sec latency 35.534 ms 2024-04-12T23:10:55.900 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:55 smithi144 ceph-mon[67032]: pgmap v129: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s; 539 KiB/s, 20 objects/s recovering 2024-04-12T23:10:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:55 smithi181 ceph-mon[52974]: pgmap v129: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.1 MiB/s wr, 505 op/s; 539 KiB/s, 20 objects/s recovering 2024-04-12T23:10:56.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 227496 10.89 MB/sec execute 377 sec latency 44.291 ms 2024-04-12T23:10:57.216 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:57.216 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:57.216 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:57.216 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:57 smithi144 ceph-mon[67032]: pgmap v130: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 465 op/s; 366 KiB/s, 15 objects/s recovering 2024-04-12T23:10:57.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 227751 10.89 MB/sec execute 378 sec latency 60.841 ms 2024-04-12T23:10:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:10:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:10:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:57 smithi181 ceph-mon[52974]: pgmap v130: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.9 MiB/s wr, 465 op/s; 366 KiB/s, 15 objects/s recovering 2024-04-12T23:10:58.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 227995 10.88 MB/sec execute 379 sec latency 105.559 ms 2024-04-12T23:10:59.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 228200 10.87 MB/sec execute 380 sec latency 71.553 ms 2024-04-12T23:10:59.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:10:59 smithi144 ceph-mon[67032]: pgmap v131: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.6 MiB/s wr, 660 op/s; 341 KiB/s, 14 objects/s recovering 2024-04-12T23:10:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:10:59 smithi181 ceph-mon[52974]: pgmap v131: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.6 MiB/s wr, 660 op/s; 341 KiB/s, 14 objects/s recovering 2024-04-12T23:11:00.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 228401 10.86 MB/sec execute 381 sec latency 63.319 ms 2024-04-12T23:11:00.804 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:00 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:11:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:00 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:11:01.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 228688 10.84 MB/sec execute 382 sec latency 54.098 ms 2024-04-12T23:11:01.814 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:01 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:11:01.814 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:01 smithi144 ceph-mon[67032]: Upgrade: osd.2 is safe to restart 2024-04-12T23:11:01.814 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:01 smithi144 ceph-mon[67032]: pgmap v132: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-04-12T23:11:01.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:01 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-12T23:11:01.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:01 smithi181 ceph-mon[52974]: Upgrade: osd.2 is safe to restart 2024-04-12T23:11:01.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:01 smithi181 ceph-mon[52974]: pgmap v132: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-04-12T23:11:01.980 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:11:02.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 228999 10.82 MB/sec execute 383 sec latency 42.170 ms 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (5m) 25s ago 14m 16.1M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (2m) 25s ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (116s) 111s ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 25s ago 14m - - 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (8m) 25s ago 8m 38.4M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (8m) 111s ago 8m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:11:02.931 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (4m) 25s ago 16m 505M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (3m) 111s ago 13m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (3m) 25s ago 16m 49.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (3m) 111s ago 13m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (8m) 25s ago 8m 271M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (8m) 111s ago 8m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (5m) 25s ago 14m 11.0M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (4m) 111s ago 13m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (84s) 25s ago 13m 253M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (35s) 25s ago 12m 12.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (12m) 25s ago 12m 566M 1936M 17.2.0 e1d6a67b021e cbe8a462f7e7 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (11m) 25s ago 11m 401M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (11m) 111s ago 11m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (11m) 111s ago 11m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (10m) 111s ago 10m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:11:02.932 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (10m) 111s ago 10m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:11:02.933 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (3m) 25s ago 13m 60.5M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:11:03.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 229212 10.80 MB/sec execute 384 sec latency 56.385 ms 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:03.643 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 6 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:11:03.644 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: from='client.44221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: pgmap v133: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 655 op/s 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: Upgrade: Updating osd.2 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: Deploying daemon osd.2 on smithi144 2024-04-12T23:11:03.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:03 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/4055779324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:03.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: from='client.44221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: pgmap v133: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 655 op/s 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: Upgrade: Updating osd.2 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: Deploying daemon osd.2 on smithi144 2024-04-12T23:11:03.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:03 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/4055779324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:04.168 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "8/23 daemons upgraded", 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:11:04.169 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:11:04.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 229487 10.80 MB/sec execute 385 sec latency 53.087 ms 2024-04-12T23:11:04.828 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:11:04.828 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:11:04.829 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:11:04.911 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:04 smithi144 ceph-mon[67032]: from='client.44225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:04 smithi181 ceph-mon[52974]: from='client.44225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:05.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 229727 10.79 MB/sec execute 386 sec latency 67.088 ms 2024-04-12T23:11:05.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:05 smithi144 ceph-mon[67032]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:05.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:05 smithi144 ceph-mon[67032]: pgmap v134: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 520 op/s 2024-04-12T23:11:05.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:05 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2084631306' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:11:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:05 smithi181 ceph-mon[52974]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:05 smithi181 ceph-mon[52974]: pgmap v134: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 520 op/s 2024-04-12T23:11:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:05 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2084631306' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:11:06.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 230033 10.78 MB/sec execute 387 sec latency 77.748 ms 2024-04-12T23:11:07.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 230403 10.77 MB/sec execute 388 sec latency 75.752 ms 2024-04-12T23:11:07.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:07 smithi144 ceph-mon[67032]: pgmap v135: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-04-12T23:11:07.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:07 smithi181 ceph-mon[52974]: pgmap v135: 97 pgs: 97 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-04-12T23:11:08.217 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:08 smithi144 systemd[1]: Stopping Ceph osd.2 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:11:08.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 230590 10.76 MB/sec execute 389 sec latency 75.123 ms 2024-04-12T23:11:08.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:08 smithi144 ceph-mon[67032]: osd.2 marked itself down and dead 2024-04-12T23:11:08.773 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:08 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[43017]: 2024-04-12T23:11:08.512+0000 7ff5e3193700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:11:08.774 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:08 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[43017]: 2024-04-12T23:11:08.512+0000 7ff5e3193700 -1 osd.2 99 *** Got signal Terminated *** 2024-04-12T23:11:08.774 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:08 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[43017]: 2024-04-12T23:11:08.512+0000 7ff5e3193700 -1 osd.2 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:11:08.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:08 smithi181 ceph-mon[52974]: osd.2 marked itself down and dead 2024-04-12T23:11:09.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 230628 10.73 MB/sec execute 390 sec latency 794.734 ms 2024-04-12T23:11:09.600 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:09 smithi144 podman[78299]: 2024-04-12 23:11:09.315379117 +0000 UTC m=+0.945611935 container died cbe8a462f7e7acf1454027a9ccaba73e218cdf9877a9b23c0affe61a0290efff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, RELEASE=HEAD, architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux ) 2024-04-12T23:11:09.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:09 smithi144 ceph-mon[67032]: pgmap v136: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 559 op/s 2024-04-12T23:11:09.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:09 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:11:09.851 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:09 smithi144 ceph-mon[67032]: osdmap e100: 8 total, 7 up, 8 in 2024-04-12T23:11:09.851 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:09 smithi144 podman[78299]: 2024-04-12 23:11:09.803772958 +0000 UTC m=+1.434005778 container cleanup cbe8a462f7e7acf1454027a9ccaba73e218cdf9877a9b23c0affe61a0290efff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, ceph=True, io.openshift.expose-services=, distribution-scope=public, GIT_BRANCH=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-12T23:11:09.851 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:09 smithi144 bash[78299]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2 2024-04-12T23:11:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:09 smithi181 ceph-mon[52974]: pgmap v136: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 559 op/s 2024-04-12T23:11:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:09 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:11:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:09 smithi181 ceph-mon[52974]: osdmap e100: 8 total, 7 up, 8 in 2024-04-12T23:11:10.357 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:10 smithi144 podman[78317]: 2024-04-12 23:11:10.275077879 +0000 UTC m=+0.951670407 container remove cbe8a462f7e7acf1454027a9ccaba73e218cdf9877a9b23c0affe61a0290efff (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, vcs-type=git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8) 2024-04-12T23:11:10.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 230965 10.74 MB/sec execute 391 sec latency 1060.895 ms 2024-04-12T23:11:10.862 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:10 smithi144 ceph-mon[67032]: osdmap e101: 8 total, 7 up, 8 in 2024-04-12T23:11:10.862 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:10 smithi144 podman[78405]: 2024-04-12 23:11:10.644507587 +0000 UTC m=+0.029056582 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:11:10.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:10 smithi181 ceph-mon[52974]: osdmap e101: 8 total, 7 up, 8 in 2024-04-12T23:11:11.113 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:10 smithi144 podman[78405]: 2024-04-12 23:11:10.991845161 +0000 UTC m=+0.376394151 container create 59f9956e63ecac17d704f914bc9e7d4bd9b6db4e401fcf32a09de350bd65a11b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T23:11:11.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 231464 10.76 MB/sec execute 392 sec latency 25.304 ms 2024-04-12T23:11:11.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:11 smithi144 ceph-mon[67032]: pgmap v139: 97 pgs: 1 active+undersized, 1 active+undersized+degraded, 5 stale+active+clean, 90 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.5 MiB/s wr, 278 op/s; 11/1569 objects degraded (0.701%) 2024-04-12T23:11:11.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:11 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 11/1569 objects degraded (0.701%), 1 pg degraded (PG_DEGRADED) 2024-04-12T23:11:11.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:11.866 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:11.866 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:11 smithi144 podman[78405]: 2024-04-12 23:11:11.666951634 +0000 UTC m=+1.051500635 container init 59f9956e63ecac17d704f914bc9e7d4bd9b6db4e401fcf32a09de350bd65a11b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T23:11:11.867 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:11 smithi144 podman[78405]: 2024-04-12 23:11:11.680086675 +0000 UTC m=+1.064635664 container start 59f9956e63ecac17d704f914bc9e7d4bd9b6db4e401fcf32a09de350bd65a11b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb) 2024-04-12T23:11:11.867 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:11 smithi144 podman[78405]: 2024-04-12 23:11:11.855627474 +0000 UTC m=+1.240176464 container attach 59f9956e63ecac17d704f914bc9e7d4bd9b6db4e401fcf32a09de350bd65a11b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b277feb, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-12T23:11:11.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:11 smithi181 ceph-mon[52974]: pgmap v139: 97 pgs: 1 active+undersized, 1 active+undersized+degraded, 5 stale+active+clean, 90 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.5 MiB/s wr, 278 op/s; 11/1569 objects degraded (0.701%) 2024-04-12T23:11:11.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:11 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 11/1569 objects degraded (0.701%), 1 pg degraded (PG_DEGRADED) 2024-04-12T23:11:11.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:11.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:12.380 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:12 smithi144 podman[78405]: 2024-04-12 23:11:12.144043868 +0000 UTC m=+1.528592860 container died 59f9956e63ecac17d704f914bc9e7d4bd9b6db4e401fcf32a09de350bd65a11b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:11:12.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 231825 10.76 MB/sec execute 393 sec latency 31.485 ms 2024-04-12T23:11:13.396 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:13 smithi144 podman[78429]: 2024-04-12 23:11:13.264810294 +0000 UTC m=+1.107312265 container remove 59f9956e63ecac17d704f914bc9e7d4bd9b6db4e401fcf32a09de350bd65a11b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:11:13.396 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:13 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-12T23:11:13.396 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:13 smithi144 systemd[1]: Stopped Ceph osd.2 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:11:13.397 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:13 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2.service: Consumed 3min 6.816s CPU time. 2024-04-12T23:11:13.423 INFO:teuthology.orchestra.run.smithi144.stdout: 5 232076 10.74 MB/sec execute 394 sec latency 63.128 ms 2024-04-12T23:11:13.653 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:13 smithi144 systemd[1]: Starting Ceph osd.2 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:11:13.905 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:13 smithi144 ceph-mon[67032]: pgmap v140: 97 pgs: 3 active+undersized, 17 active+undersized+degraded, 2 stale+active+clean, 75 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 121/1668 objects degraded (7.254%) 2024-04-12T23:11:13.905 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:13 smithi144 podman[78520]: 2024-04-12 23:11:13.80961447 +0000 UTC m=+0.025632537 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:11:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:13 smithi181 ceph-mon[52974]: pgmap v140: 97 pgs: 3 active+undersized, 17 active+undersized+degraded, 2 stale+active+clean, 75 active+clean; 340 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 385 op/s; 121/1668 objects degraded (7.254%) 2024-04-12T23:11:14.414 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:14 smithi144 podman[78520]: 2024-04-12 23:11:14.286546273 +0000 UTC m=+0.502564330 container create 2083d8b7565a0f491d023e2d441a7af85d62f49c47808d14464dfdfe9aa5f8ba (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:11:14.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 232414 10.72 MB/sec execute 395 sec latency 36.352 ms 2024-04-12T23:11:14.665 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:14 smithi144 podman[78520]: 2024-04-12 23:11:14.672117248 +0000 UTC m=+0.888135300 container init 2083d8b7565a0f491d023e2d441a7af85d62f49c47808d14464dfdfe9aa5f8ba (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-12T23:11:14.665 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:14 smithi144 podman[78520]: 2024-04-12 23:11:14.677280973 +0000 UTC m=+0.893299025 container start 2083d8b7565a0f491d023e2d441a7af85d62f49c47808d14464dfdfe9aa5f8ba (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:11:14.920 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:14 smithi144 podman[78520]: 2024-04-12 23:11:14.834630978 +0000 UTC m=+1.050649043 container attach 2083d8b7565a0f491d023e2d441a7af85d62f49c47808d14464dfdfe9aa5f8ba (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, ceph=True, GIT_CLEAN=True) 2024-04-12T23:11:15.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 232677 10.71 MB/sec execute 396 sec latency 65.690 ms 2024-04-12T23:11:15.424 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T23:11:15.424 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 bash[78520]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T23:11:15.425 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T23:11:15.426 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-12T23:11:15.426 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-12T23:11:15.426 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate[78533]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-12T23:11:15.426 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:15 smithi144 podman[78520]: 2024-04-12 23:11:15.353072269 +0000 UTC m=+1.569090313 container died 2083d8b7565a0f491d023e2d441a7af85d62f49c47808d14464dfdfe9aa5f8ba (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-12T23:11:15.933 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:15 smithi144 ceph-mon[67032]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 559 op/s; 154/1665 objects degraded (9.249%) 2024-04-12T23:11:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:15 smithi181 ceph-mon[52974]: pgmap v141: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 559 op/s; 154/1665 objects degraded (9.249%) 2024-04-12T23:11:16.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 233048 10.71 MB/sec execute 397 sec latency 46.691 ms 2024-04-12T23:11:16.715 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:16 smithi144 podman[78669]: 2024-04-12 23:11:16.605902418 +0000 UTC m=+1.246639349 container remove 2083d8b7565a0f491d023e2d441a7af85d62f49c47808d14464dfdfe9aa5f8ba (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-activate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb) 2024-04-12T23:11:16.967 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:16 smithi144 ceph-mon[67032]: pgmap v142: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.7 MiB/s wr, 334 op/s; 157/1689 objects degraded (9.295%) 2024-04-12T23:11:16.967 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:16 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 157/1689 objects degraded (9.295%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:16.967 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:16 smithi144 podman[78758]: 2024-04-12 23:11:16.74653757 +0000 UTC m=+0.024079949 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:11:16.967 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:16 smithi144 podman[78758]: 2024-04-12 23:11:16.923685415 +0000 UTC m=+0.201227774 container create 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-12T23:11:17.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:16 smithi181 ceph-mon[52974]: pgmap v142: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.7 MiB/s wr, 334 op/s; 157/1689 objects degraded (9.295%) 2024-04-12T23:11:17.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:16 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 157/1689 objects degraded (9.295%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:17.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 233405 10.71 MB/sec execute 398 sec latency 50.755 ms 2024-04-12T23:11:17.985 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:17 smithi144 podman[78758]: 2024-04-12 23:11:17.9367761 +0000 UTC m=+1.214318467 container init 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:11:17.985 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:17 smithi144 podman[78758]: 2024-04-12 23:11:17.942623642 +0000 UTC m=+1.220166005 container start 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:11:17.985 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:17 smithi144 ceph-osd[78776]: -- 172.21.15.144:0/1373305670 <== mon.0 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555ccc257680 con 0x555ccc221000 2024-04-12T23:11:18.237 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:18 smithi144 bash[78758]: 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 2024-04-12T23:11:18.237 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:18 smithi144 systemd[1]: Started Ceph osd.2 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:11:18.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 233805 10.70 MB/sec execute 399 sec latency 63.898 ms 2024-04-12T23:11:18.753 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:11:18.533+0000 7f39ac25d7c0 -1 Falling back to public interface 2024-04-12T23:11:19.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:19 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:19.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:19 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:19.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:19 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:11:19.258 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:19 smithi144 ceph-mon[67032]: pgmap v143: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 458 op/s; 146/1530 objects degraded (9.542%) 2024-04-12T23:11:19.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 234142 10.70 MB/sec execute 400 sec latency 44.982 ms 2024-04-12T23:11:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:19 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:19 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:19 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:11:19.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:19 smithi181 ceph-mon[52974]: pgmap v143: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 458 op/s; 146/1530 objects degraded (9.542%) 2024-04-12T23:11:20.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 234471 10.71 MB/sec execute 401 sec latency 37.102 ms 2024-04-12T23:11:21.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 234854 10.71 MB/sec execute 402 sec latency 33.764 ms 2024-04-12T23:11:21.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:21 smithi144 ceph-mon[67032]: pgmap v144: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 438 op/s; 146/1557 objects degraded (9.377%) 2024-04-12T23:11:21.816 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:21 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:21 smithi181 ceph-mon[52974]: pgmap v144: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 438 op/s; 146/1557 objects degraded (9.377%) 2024-04-12T23:11:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:21 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:22.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 235240 10.72 MB/sec execute 403 sec latency 37.954 ms 2024-04-12T23:11:22.830 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:22 smithi144 ceph-mon[67032]: pgmap v145: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 147/1656 objects degraded (8.877%) 2024-04-12T23:11:22.831 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:22 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 147/1656 objects degraded (8.877%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:23.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:22 smithi181 ceph-mon[52974]: pgmap v145: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 147/1656 objects degraded (8.877%) 2024-04-12T23:11:23.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:22 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 147/1656 objects degraded (8.877%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:23.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 235568 10.71 MB/sec execute 404 sec latency 38.906 ms 2024-04-12T23:11:24.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 235990 10.70 MB/sec execute 405 sec latency 45.495 ms 2024-04-12T23:11:24.978 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:24 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:11:24.861+0000 7f39ac25d7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-12T23:11:25.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 236320 10.69 MB/sec execute 406 sec latency 46.107 ms 2024-04-12T23:11:25.734 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:25 smithi144 ceph-mon[67032]: pgmap v146: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 502 op/s; 149/1677 objects degraded (8.885%) 2024-04-12T23:11:25.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:25 smithi181 ceph-mon[52974]: pgmap v146: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 502 op/s; 149/1677 objects degraded (8.885%) 2024-04-12T23:11:26.247 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:26 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:11:26.128+0000 7f39ac25d7c0 -1 osd.2 99 log_to_monitors true 2024-04-12T23:11:26.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 236679 10.69 MB/sec execute 407 sec latency 66.551 ms 2024-04-12T23:11:26.754 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:26.754 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:26 smithi144 ceph-mon[67032]: from='osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T23:11:26.754 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:26.754 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:26.754 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:11:26 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:11:26.732+0000 7f399d6b5700 -1 osd.2 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:11:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:26 smithi181 ceph-mon[52974]: from='osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-12T23:11:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:27.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 237089 10.69 MB/sec execute 408 sec latency 43.814 ms 2024-04-12T23:11:28.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:27 smithi144 ceph-mon[67032]: pgmap v147: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 394 op/s; 146/1668 objects degraded (8.753%) 2024-04-12T23:11:28.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:27 smithi144 ceph-mon[67032]: from='osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T23:11:28.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:27 smithi144 ceph-mon[67032]: osdmap e102: 8 total, 7 up, 8 in 2024-04-12T23:11:28.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:27 smithi144 ceph-mon[67032]: from='osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:11:28.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:27 smithi181 ceph-mon[52974]: pgmap v147: 97 pgs: 6 active+undersized, 20 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 394 op/s; 146/1668 objects degraded (8.753%) 2024-04-12T23:11:28.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:27 smithi181 ceph-mon[52974]: from='osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-12T23:11:28.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:27 smithi181 ceph-mon[52974]: osdmap e102: 8 total, 7 up, 8 in 2024-04-12T23:11:28.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:27 smithi181 ceph-mon[52974]: from='osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:11:28.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 237233 10.66 MB/sec execute 409 sec latency 730.207 ms 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139] boot 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: pgmap v150: 97 pgs: 4 peering, 4 active+undersized, 18 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 648 op/s; 114/1590 objects degraded (7.170%) 2024-04-12T23:11:28.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:28 smithi144 ceph-mon[67032]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T23:11:29.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:11:29.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: osd.2 [v2:172.21.15.144:6818/1495857139,v1:172.21.15.144:6819/1495857139] boot 2024-04-12T23:11:29.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: osdmap e103: 8 total, 8 up, 8 in 2024-04-12T23:11:29.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-12T23:11:29.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:29.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:29.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: pgmap v150: 97 pgs: 4 peering, 4 active+undersized, 18 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 648 op/s; 114/1590 objects degraded (7.170%) 2024-04-12T23:11:29.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:28 smithi181 ceph-mon[52974]: osdmap e104: 8 total, 8 up, 8 in 2024-04-12T23:11:29.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 237233 10.64 MB/sec execute 410 sec latency 1730.270 ms 2024-04-12T23:11:29.838 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:29 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 114/1590 objects degraded (7.170%), 18 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:29.838 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:29 smithi144 ceph-mon[67032]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T23:11:30.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:29 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 114/1590 objects degraded (7.170%), 18 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:30.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:29 smithi181 ceph-mon[52974]: osdmap e105: 8 total, 8 up, 8 in 2024-04-12T23:11:30.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 237257 10.61 MB/sec execute 411 sec latency 2583.173 ms 2024-04-12T23:11:31.414 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:31 smithi144 ceph-mon[67032]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T23:11:31.414 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:31 smithi144 ceph-mon[67032]: pgmap v154: 97 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 3 active+undersized, 17 active+undersized+degraded, 72 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 5.9 MiB/s wr, 707 op/s; 125/1548 objects degraded (8.075%) 2024-04-12T23:11:31.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 237684 10.62 MB/sec execute 412 sec latency 48.420 ms 2024-04-12T23:11:31.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:31 smithi181 ceph-mon[52974]: osdmap e106: 8 total, 8 up, 8 in 2024-04-12T23:11:31.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:31 smithi181 ceph-mon[52974]: pgmap v154: 97 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 3 active+undersized, 17 active+undersized+degraded, 72 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 5.9 MiB/s wr, 707 op/s; 125/1548 objects degraded (8.075%) 2024-04-12T23:11:32.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 238073 10.63 MB/sec execute 413 sec latency 35.515 ms 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: osdmap e107: 8 total, 8 up, 8 in 2024-04-12T23:11:32.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:11:32.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:11:32.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:32.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:33.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 238344 10.63 MB/sec execute 414 sec latency 853.416 ms 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: pgmap v156: 97 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 peering, 3 active+undersized+remapped, 86 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 KiB/s rd, 2.8 MiB/s wr, 221 op/s; 34/1611 objects degraded (2.110%); 3.4 MiB/s, 28 objects/s recovering 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: pgmap v157: 97 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 peering, 3 active+undersized+remapped, 86 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 KiB/s rd, 2.9 MiB/s wr, 225 op/s; 34/1611 objects degraded (2.110%); 3.4 MiB/s, 28 objects/s recovering 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:33.453 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T23:11:33.454 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:33 smithi144 ceph-mon[67032]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T23:11:33.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: pgmap v156: 97 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 peering, 3 active+undersized+remapped, 86 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 KiB/s rd, 2.8 MiB/s wr, 221 op/s; 34/1611 objects degraded (2.110%); 3.4 MiB/s, 28 objects/s recovering 2024-04-12T23:11:33.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: pgmap v157: 97 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 6 peering, 3 active+undersized+remapped, 86 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 KiB/s rd, 2.9 MiB/s wr, 225 op/s; 34/1611 objects degraded (2.110%); 3.4 MiB/s, 28 objects/s recovering 2024-04-12T23:11:33.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: osdmap e108: 8 total, 8 up, 8 in 2024-04-12T23:11:33.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-12T23:11:33.698 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:33 smithi181 ceph-mon[52974]: osdmap e109: 8 total, 8 up, 8 in 2024-04-12T23:11:34.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 238760 10.64 MB/sec execute 415 sec latency 860.950 ms 2024-04-12T23:11:34.711 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:34 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 34/1593 objects degraded (2.134%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:34.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:34 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 34/1593 objects degraded (2.134%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:11:35.350 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:11:35.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 239045 10.63 MB/sec execute 416 sec latency 55.538 ms 2024-04-12T23:11:35.724 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:35 smithi144 ceph-mon[67032]: pgmap v160: 97 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 3 peering, 3 active+undersized+remapped, 89 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 KiB/s rd, 3.8 MiB/s wr, 473 op/s; 34/1593 objects degraded (2.134%); 3.8 MiB/s, 45 objects/s recovering 2024-04-12T23:11:35.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:35 smithi181 ceph-mon[52974]: pgmap v160: 97 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 3 peering, 3 active+undersized+remapped, 89 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 1.1 KiB/s rd, 3.8 MiB/s wr, 473 op/s; 34/1593 objects degraded (2.134%); 3.8 MiB/s, 45 objects/s recovering 2024-04-12T23:11:36.300 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:11:36.300 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (6m) 9s ago 15m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:11:36.300 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (2m) 9s ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:11:36.300 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (2m) 2m ago 14m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 9s ago 14m - - 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (9m) 9s ago 9m 38.4M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (9m) 2m ago 9m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (5m) 9s ago 16m 507M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (4m) 9s ago 16m 49.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (3m) 2m ago 14m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (9m) 9s ago 9m 265M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (9m) 2m ago 9m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (5m) 9s ago 14m 13.4M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (5m) 2m ago 14m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (117s) 9s ago 13m 310M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (68s) 9s ago 13m 256M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (19s) 9s ago 12m 12.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (12m) 9s ago 12m 427M 1936M 17.2.0 e1d6a67b021e 92a11e631c6b 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (12m) 2m ago 12m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:11:36.301 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (11m) 2m ago 11m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:11:36.302 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (11m) 2m ago 11m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:11:36.302 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (10m) 2m ago 10m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:11:36.302 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (4m) 9s ago 14m 62.8M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:11:36.424 INFO:teuthology.orchestra.run.smithi144.stdout: 5 239291 10.61 MB/sec execute 417 sec latency 66.151 ms 2024-04-12T23:11:36.497 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:36 smithi144 ceph-mon[67032]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:36.847 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:11:36.847 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:11:36.848 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-12T23:11:36.849 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 7 2024-04-12T23:11:36.849 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:11:36.849 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:11:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:36 smithi181 ceph-mon[52974]: from='client.34302 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:37.353 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:11:37.353 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:11:37.353 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:11:37.353 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:11:37.353 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:11:37.353 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "9/23 daemons upgraded", 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:11:37.354 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:11:37.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 239514 10.60 MB/sec execute 418 sec latency 67.017 ms 2024-04-12T23:11:37.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:37 smithi144 ceph-mon[67032]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:37.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:37 smithi144 ceph-mon[67032]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:37.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:37 smithi144 ceph-mon[67032]: pgmap v161: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 3 active+undersized+remapped, 90 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 839 B/s rd, 4.0 MiB/s wr, 389 op/s; 12/1614 objects degraded (0.743%); 2.8 MiB/s, 38 objects/s recovering 2024-04-12T23:11:37.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:37 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/4041044980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:37 smithi181 ceph-mon[52974]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:37 smithi181 ceph-mon[52974]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:37 smithi181 ceph-mon[52974]: pgmap v161: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 3 active+undersized+remapped, 90 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 839 B/s rd, 4.0 MiB/s wr, 389 op/s; 12/1614 objects degraded (0.743%); 2.8 MiB/s, 38 objects/s recovering 2024-04-12T23:11:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:37 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/4041044980' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:11:37.956 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 12/1614 objects degraded (0.743%), 1 pg degraded 2024-04-12T23:11:37.956 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:11:37.956 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:11:37.956 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 12/1614 objects degraded (0.743%), 1 pg degraded 2024-04-12T23:11:37.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.9 is active+recovery_wait+degraded, acting [4,2,7] 2024-04-12T23:11:38.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 239768 10.58 MB/sec execute 419 sec latency 72.704 ms 2024-04-12T23:11:38.554 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:38 smithi144 ceph-mon[67032]: from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:38.555 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:38 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/4125464123' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:11:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:38 smithi181 ceph-mon[52974]: from='client.34318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:11:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:38 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/4125464123' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:11:39.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 240055 10.58 MB/sec execute 420 sec latency 48.099 ms 2024-04-12T23:11:39.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:39 smithi144 ceph-mon[67032]: pgmap v162: 97 pgs: 1 peering, 96 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 440 op/s; 261 KiB/s, 16 objects/s recovering 2024-04-12T23:11:39.577 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:39 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/1614 objects degraded (0.743%), 1 pg degraded) 2024-04-12T23:11:39.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:39 smithi181 ceph-mon[52974]: pgmap v162: 97 pgs: 1 peering, 96 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 440 op/s; 261 KiB/s, 16 objects/s recovering 2024-04-12T23:11:39.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:39 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12/1614 objects degraded (0.743%), 1 pg degraded) 2024-04-12T23:11:40.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 240425 10.58 MB/sec execute 421 sec latency 44.885 ms 2024-04-12T23:11:41.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 240735 10.57 MB/sec execute 422 sec latency 58.213 ms 2024-04-12T23:11:41.630 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:41 smithi144 ceph-mon[67032]: pgmap v163: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.2 MiB/s wr, 428 op/s; 196 KiB/s, 12 objects/s recovering 2024-04-12T23:11:41.631 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:41.631 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:41.749 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:41 smithi181 ceph-mon[52974]: pgmap v163: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.2 MiB/s wr, 428 op/s; 196 KiB/s, 12 objects/s recovering 2024-04-12T23:11:41.749 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:41.749 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:42.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 240992 10.55 MB/sec execute 423 sec latency 76.401 ms 2024-04-12T23:11:43.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 241252 10.54 MB/sec execute 424 sec latency 83.066 ms 2024-04-12T23:11:43.666 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:43 smithi144 ceph-mon[67032]: pgmap v164: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 430 op/s; 159 KiB/s, 9 objects/s recovering 2024-04-12T23:11:43.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:43 smithi181 ceph-mon[52974]: pgmap v164: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 430 op/s; 159 KiB/s, 9 objects/s recovering 2024-04-12T23:11:44.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 241478 10.54 MB/sec execute 425 sec latency 45.731 ms 2024-04-12T23:11:45.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 241750 10.54 MB/sec execute 426 sec latency 48.349 ms 2024-04-12T23:11:45.738 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:45 smithi144 ceph-mon[67032]: pgmap v165: 97 pgs: 97 active+clean; 326 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 375 op/s; 2.6 KiB/s, 3 objects/s recovering 2024-04-12T23:11:45.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:45 smithi181 ceph-mon[52974]: pgmap v165: 97 pgs: 97 active+clean; 326 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.9 MiB/s wr, 375 op/s; 2.6 KiB/s, 3 objects/s recovering 2024-04-12T23:11:46.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 242073 10.54 MB/sec execute 427 sec latency 52.283 ms 2024-04-12T23:11:47.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:47 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:47.307 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:47 smithi144 ceph-mon[67032]: pgmap v166: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 351 op/s; 2.3 KiB/s, 2 objects/s recovering 2024-04-12T23:11:47.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 242417 10.55 MB/sec execute 428 sec latency 42.119 ms 2024-04-12T23:11:47.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:47 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:47.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:47 smithi181 ceph-mon[52974]: pgmap v166: 97 pgs: 97 active+clean; 327 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 351 op/s; 2.3 KiB/s, 2 objects/s recovering 2024-04-12T23:11:48.116 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:48.116 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:48 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:48.116 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:48 smithi144 ceph-mon[67032]: Upgrade: osd.3 is safe to restart 2024-04-12T23:11:48.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 242650 10.53 MB/sec execute 429 sec latency 96.476 ms 2024-04-12T23:11:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:48 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-12T23:11:48.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:48 smithi181 ceph-mon[52974]: Upgrade: osd.3 is safe to restart 2024-04-12T23:11:49.134 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:49 smithi144 ceph-mon[67032]: pgmap v167: 97 pgs: 97 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.7 MiB/s wr, 456 op/s; 0 B/s, 0 objects/s recovering 2024-04-12T23:11:49.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 242949 10.52 MB/sec execute 430 sec latency 41.635 ms 2024-04-12T23:11:49.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:49 smithi181 ceph-mon[52974]: pgmap v167: 97 pgs: 97 active+clean; 330 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.7 MiB/s wr, 456 op/s; 0 B/s, 0 objects/s recovering 2024-04-12T23:11:50.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 243464 10.52 MB/sec execute 431 sec latency 34.533 ms 2024-04-12T23:11:51.166 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:51 smithi144 ceph-mon[67032]: Upgrade: Updating osd.3 2024-04-12T23:11:51.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:51.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T23:11:51.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:11:51.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:51 smithi144 ceph-mon[67032]: Deploying daemon osd.3 on smithi144 2024-04-12T23:11:51.167 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:51 smithi144 ceph-mon[67032]: pgmap v168: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-04-12T23:11:51.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 243947 10.53 MB/sec execute 432 sec latency 51.928 ms 2024-04-12T23:11:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:51 smithi181 ceph-mon[52974]: Upgrade: Updating osd.3 2024-04-12T23:11:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:11:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-12T23:11:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:11:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:51 smithi181 ceph-mon[52974]: Deploying daemon osd.3 on smithi144 2024-04-12T23:11:51.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:51 smithi181 ceph-mon[52974]: pgmap v168: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-04-12T23:11:52.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 244307 10.52 MB/sec execute 433 sec latency 96.985 ms 2024-04-12T23:11:53.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 244669 10.51 MB/sec execute 434 sec latency 35.062 ms 2024-04-12T23:11:53.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:53 smithi144 ceph-mon[67032]: pgmap v169: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-04-12T23:11:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:53 smithi181 ceph-mon[52974]: pgmap v169: 97 pgs: 97 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-04-12T23:11:54.425 INFO:teuthology.orchestra.run.smithi144.stdout: 5 244919 10.50 MB/sec execute 435 sec latency 42.116 ms 2024-04-12T23:11:55.432 INFO:teuthology.orchestra.run.smithi144.stdout: 5 245313 10.52 MB/sec execute 436 sec latency 50.720 ms 2024-04-12T23:11:55.471 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:55 smithi144 ceph-mon[67032]: pgmap v170: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 466 op/s 2024-04-12T23:11:55.472 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:55 smithi144 systemd[1]: Stopping Ceph osd.3 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:11:55.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:55 smithi181 ceph-mon[52974]: pgmap v170: 97 pgs: 97 active+clean; 336 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 466 op/s 2024-04-12T23:11:56.009 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:55 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[46279]: 2024-04-12T23:11:55.883+0000 7f3bc818c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:11:56.009 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:55 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[46279]: 2024-04-12T23:11:55.883+0000 7f3bc818c700 -1 osd.3 109 *** Got signal Terminated *** 2024-04-12T23:11:56.009 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:55 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[46279]: 2024-04-12T23:11:55.883+0000 7f3bc818c700 -1 osd.3 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:11:56.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 245617 10.52 MB/sec execute 437 sec latency 79.108 ms 2024-04-12T23:11:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:56 smithi181 ceph-mon[52974]: osd.3 marked itself down and dead 2024-04-12T23:11:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:56 smithi144 ceph-mon[67032]: osd.3 marked itself down and dead 2024-04-12T23:11:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:11:57.367 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:57 smithi144 podman[80841]: 2024-04-12 23:11:57.065706273 +0000 UTC m=+1.365429268 container died 92a11e631c6bb0ecc4482c3b0cb63b1ac335d2d6c260fd6f459df2dacad15b12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, build-date=2022-05-03T08:36:31.336870, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-12T23:11:57.432 INFO:teuthology.orchestra.run.smithi144.stdout: 5 245628 10.49 MB/sec execute 438 sec latency 1052.642 ms 2024-04-12T23:11:57.621 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:57 smithi144 ceph-mon[67032]: pgmap v171: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 413 op/s 2024-04-12T23:11:57.622 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:57 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:11:57.622 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:57 smithi144 ceph-mon[67032]: osdmap e110: 8 total, 7 up, 8 in 2024-04-12T23:11:57.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:57 smithi181 ceph-mon[52974]: pgmap v171: 97 pgs: 97 active+clean; 334 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 413 op/s 2024-04-12T23:11:57.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:57 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:11:57.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:57 smithi181 ceph-mon[52974]: osdmap e110: 8 total, 7 up, 8 in 2024-04-12T23:11:58.154 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:58 smithi144 podman[80841]: 2024-04-12 23:11:58.116288276 +0000 UTC m=+2.416011262 container cleanup 92a11e631c6bb0ecc4482c3b0cb63b1ac335d2d6c260fd6f459df2dacad15b12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, GIT_CLEAN=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, release=754, CEPH_POINT_RELEASE=-17.2.0, version=8, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T23:11:58.154 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:58 smithi144 bash[80841]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3 2024-04-12T23:11:58.404 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:58 smithi144 ceph-mon[67032]: osdmap e111: 8 total, 7 up, 8 in 2024-04-12T23:11:58.432 INFO:teuthology.orchestra.run.smithi144.stdout: 5 245996 10.50 MB/sec execute 439 sec latency 48.506 ms 2024-04-12T23:11:58.657 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:58 smithi144 podman[80890]: 2024-04-12 23:11:58.613821034 +0000 UTC m=+1.542355271 container remove 92a11e631c6bb0ecc4482c3b0cb63b1ac335d2d6c260fd6f459df2dacad15b12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., name=centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, release=754, io.buildah.version=1.19.8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-04-12T23:11:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:58 smithi181 ceph-mon[52974]: osdmap e111: 8 total, 7 up, 8 in 2024-04-12T23:11:59.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:59 smithi144 ceph-mon[67032]: pgmap v174: 97 pgs: 23 peering, 6 stale+active+clean, 68 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 539 op/s 2024-04-12T23:11:59.410 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:11:59 smithi144 ceph-mon[67032]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-12T23:11:59.410 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:59 smithi144 podman[80975]: 2024-04-12 23:11:59.100734611 +0000 UTC m=+0.023460985 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:11:59.410 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:59 smithi144 podman[80975]: 2024-04-12 23:11:59.243683468 +0000 UTC m=+0.166409827 container create 3ad4ca78c97bb2f13dae0c3277b60152f26f1d2ddcb66a9344fe4a5cd32ead4b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-12T23:11:59.432 INFO:teuthology.orchestra.run.smithi144.stdout: 5 246305 10.49 MB/sec execute 440 sec latency 50.032 ms 2024-04-12T23:11:59.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:59 smithi181 ceph-mon[52974]: pgmap v174: 97 pgs: 23 peering, 6 stale+active+clean, 68 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 539 op/s 2024-04-12T23:11:59.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:11:59 smithi181 ceph-mon[52974]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-04-12T23:11:59.914 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:59 smithi144 podman[80975]: 2024-04-12 23:11:59.864416839 +0000 UTC m=+0.787143204 container init 3ad4ca78c97bb2f13dae0c3277b60152f26f1d2ddcb66a9344fe4a5cd32ead4b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409) 2024-04-12T23:11:59.914 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:11:59 smithi144 podman[80975]: 2024-04-12 23:11:59.874100413 +0000 UTC m=+0.796826772 container start 3ad4ca78c97bb2f13dae0c3277b60152f26f1d2ddcb66a9344fe4a5cd32ead4b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-12T23:12:00.179 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:00 smithi144 podman[80975]: 2024-04-12 23:12:00.040303127 +0000 UTC m=+0.963029483 container attach 3ad4ca78c97bb2f13dae0c3277b60152f26f1d2ddcb66a9344fe4a5cd32ead4b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T23:12:00.430 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:00 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 106/1620 objects degraded (6.543%), 13 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:00.430 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:00 smithi144 podman[80975]: 2024-04-12 23:12:00.310027418 +0000 UTC m=+1.232753776 container died 3ad4ca78c97bb2f13dae0c3277b60152f26f1d2ddcb66a9344fe4a5cd32ead4b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:12:00.432 INFO:teuthology.orchestra.run.smithi144.stdout: 5 246543 10.47 MB/sec execute 441 sec latency 63.596 ms 2024-04-12T23:12:00.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:00 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 106/1620 objects degraded (6.543%), 13 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:01.412 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 podman[81000]: 2024-04-12 23:12:01.300155754 +0000 UTC m=+0.983971347 container remove 3ad4ca78c97bb2f13dae0c3277b60152f26f1d2ddcb66a9344fe4a5cd32ead4b (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-12T23:12:01.412 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-12T23:12:01.412 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 systemd[1]: Stopped Ceph osd.3 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:12:01.412 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.3.service: Consumed 5min 6.142s CPU time. 2024-04-12T23:12:01.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 246826 10.46 MB/sec execute 442 sec latency 48.681 ms 2024-04-12T23:12:01.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:01 smithi144 ceph-mon[67032]: pgmap v175: 97 pgs: 5 active+undersized, 13 active+undersized+degraded, 23 peering, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s wr, 339 op/s; 106/1620 objects degraded (6.543%) 2024-04-12T23:12:01.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:01 smithi181 ceph-mon[52974]: pgmap v175: 97 pgs: 5 active+undersized, 13 active+undersized+degraded, 23 peering, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 3.8 MiB/s wr, 339 op/s; 106/1620 objects degraded (6.543%) 2024-04-12T23:12:01.917 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 systemd[1]: Starting Ceph osd.3 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:12:02.169 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 podman[81099]: 2024-04-12 23:12:01.836347886 +0000 UTC m=+0.026264152 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:12:02.169 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:01 smithi144 podman[81099]: 2024-04-12 23:12:01.966282902 +0000 UTC m=+0.156199149 container create 016dc1abc8ec28dc3a43425a364ce2e5d6a723b6d1a44bb0c720c3624af46543 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-12T23:12:02.420 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 podman[81099]: 2024-04-12 23:12:02.320195662 +0000 UTC m=+0.510111909 container init 016dc1abc8ec28dc3a43425a364ce2e5d6a723b6d1a44bb0c720c3624af46543 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:12:02.420 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 podman[81099]: 2024-04-12 23:12:02.327318612 +0000 UTC m=+0.517234867 container start 016dc1abc8ec28dc3a43425a364ce2e5d6a723b6d1a44bb0c720c3624af46543 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-12T23:12:02.420 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 podman[81099]: 2024-04-12 23:12:02.428476526 +0000 UTC m=+0.618392782 container attach 016dc1abc8ec28dc3a43425a364ce2e5d6a723b6d1a44bb0c720c3624af46543 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-04-12T23:12:02.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 247105 10.46 MB/sec execute 443 sec latency 46.774 ms 2024-04-12T23:12:02.672 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-mon[67032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T23:12:02.927 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 bash[81099]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate[81114]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-12T23:12:02.928 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:02 smithi144 podman[81099]: 2024-04-12 23:12:02.913423478 +0000 UTC m=+1.103339739 container died 016dc1abc8ec28dc3a43425a364ce2e5d6a723b6d1a44bb0c720c3624af46543 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:12:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:02 smithi181 ceph-mon[52974]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-04-12T23:12:03.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 247467 10.46 MB/sec execute 444 sec latency 56.517 ms 2024-04-12T23:12:03.720 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:03 smithi144 ceph-mon[67032]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 544 op/s; 215/1668 objects degraded (12.890%) 2024-04-12T23:12:03.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:03 smithi181 ceph-mon[52974]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 544 op/s; 215/1668 objects degraded (12.890%) 2024-04-12T23:12:04.266 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:04 smithi144 podman[81245]: 2024-04-12 23:12:04.245728496 +0000 UTC m=+1.326279316 container remove 016dc1abc8ec28dc3a43425a364ce2e5d6a723b6d1a44bb0c720c3624af46543 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-activate, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-12T23:12:04.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 247793 10.45 MB/sec execute 445 sec latency 76.422 ms 2024-04-12T23:12:04.519 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:04 smithi144 podman[81306]: 2024-04-12 23:12:04.381402793 +0000 UTC m=+0.023182252 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:12:04.773 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:04 smithi144 podman[81306]: 2024-04-12 23:12:04.711987662 +0000 UTC m=+0.353767107 container create f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:12:05.289 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:05 smithi144 podman[81306]: 2024-04-12 23:12:05.164264879 +0000 UTC m=+0.806044323 container init f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD) 2024-04-12T23:12:05.289 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:05 smithi144 podman[81306]: 2024-04-12 23:12:05.168302005 +0000 UTC m=+0.810081447 container start f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b277feb, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T23:12:05.289 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-osd[81325]: -- 172.21.15.144:0/3284628910 <== mon.0 v2:172.21.15.144:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56350c997680 con 0x56350c960c00 2024-04-12T23:12:05.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 248184 10.44 MB/sec execute 446 sec latency 41.109 ms 2024-04-12T23:12:05.552 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-mon[67032]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 528 op/s; 216/1686 objects degraded (12.811%) 2024-04-12T23:12:05.552 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:05.552 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:05.552 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:05.552 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 216/1686 objects degraded (12.811%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:05.552 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:05 smithi144 bash[81306]: f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 2024-04-12T23:12:05.552 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:05 smithi144 systemd[1]: Started Ceph osd.3 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:12:05.802 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:05 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:12:05.746+0000 7f36a5a3e7c0 -1 Falling back to public interface 2024-04-12T23:12:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:05 smithi181 ceph-mon[52974]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 528 op/s; 216/1686 objects degraded (12.811%) 2024-04-12T23:12:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:05 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:05 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:05 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:05 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 216/1686 objects degraded (12.811%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:06.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 248434 10.43 MB/sec execute 447 sec latency 57.162 ms 2024-04-12T23:12:07.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 248816 10.45 MB/sec execute 448 sec latency 49.027 ms 2024-04-12T23:12:07.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:07 smithi144 ceph-mon[67032]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 446 op/s; 209/1641 objects degraded (12.736%) 2024-04-12T23:12:07.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:07 smithi181 ceph-mon[52974]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 446 op/s; 209/1641 objects degraded (12.736%) 2024-04-12T23:12:08.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 249138 10.45 MB/sec execute 449 sec latency 51.978 ms 2024-04-12T23:12:08.481 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:12:09.352 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:12:09.352 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (6m) 42s ago 15m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:12:09.352 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (3m) 42s ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (3m) 2m ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 42s ago 15m - - 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (10m) 42s ago 10m 38.4M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (9m) 2m ago 9m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (5m) 42s ago 17m 507M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (4m) 2m ago 15m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (4m) 42s ago 17m 49.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (4m) 2m ago 15m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (9m) 42s ago 9m 265M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (9m) 2m ago 9m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (6m) 42s ago 15m 13.4M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 2m ago 14m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (2m) 42s ago 14m 310M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (101s) 42s ago 13m 256M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:12:09.353 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (52s) 42s ago 13m 12.5M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:12:09.354 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 starting - - - 1936M 2024-04-12T23:12:09.354 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (12m) 2m ago 12m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:12:09.354 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (12m) 2m ago 12m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:12:09.354 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (11m) 2m ago 11m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:12:09.354 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (11m) 2m ago 11m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:12:09.354 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (5m) 42s ago 14m 62.8M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:12:09.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 249479 10.46 MB/sec execute 450 sec latency 45.062 ms 2024-04-12T23:12:09.641 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:09 smithi144 ceph-mon[67032]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.0 MiB/s wr, 420 op/s; 205/1614 objects degraded (12.701%) 2024-04-12T23:12:09.641 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:09 smithi144 ceph-mon[67032]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:09.877 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 3 2024-04-12T23:12:09.878 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 7 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:12:09.879 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:12:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:09 smithi181 ceph-mon[52974]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.0 MiB/s wr, 420 op/s; 205/1614 objects degraded (12.701%) 2024-04-12T23:12:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:09 smithi181 ceph-mon[52974]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:10.370 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:12:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:12:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:12:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:12:10.370 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:12:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:12:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:12:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:12:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:12:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "9/23 daemons upgraded", 2024-04-12T23:12:10.371 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:12:10.372 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:12:10.372 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:12:10.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 249729 10.44 MB/sec execute 451 sec latency 55.112 ms 2024-04-12T23:12:10.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:10 smithi144 ceph-mon[67032]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:10.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:10 smithi144 ceph-mon[67032]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:10.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:10 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1137432070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:10.663 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:10 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 205/1614 objects degraded (12.701%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:10.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:10 smithi181 ceph-mon[52974]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:10.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:10 smithi181 ceph-mon[52974]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:10.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:10 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1137432070' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:10.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:10 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 205/1614 objects degraded (12.701%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:11.015 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 213/1629 objects degraded (13.076%), 27 pgs degraded 2024-04-12T23:12:11.015 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout: osd.3 (root=default,host=smithi144) is down 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 213/1629 objects degraded (13.076%), 27 pgs degraded 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-04-12T23:12:11.016 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-04-12T23:12:11.017 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-04-12T23:12:11.018 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-04-12T23:12:11.018 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-04-12T23:12:11.018 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-04-12T23:12:11.018 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-04-12T23:12:11.018 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-04-12T23:12:11.018 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-04-12T23:12:11.019 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-04-12T23:12:11.019 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-04-12T23:12:11.019 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-04-12T23:12:11.019 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-04-12T23:12:11.019 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-04-12T23:12:11.019 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-04-12T23:12:11.020 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-04-12T23:12:11.020 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-04-12T23:12:11.020 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-04-12T23:12:11.020 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-04-12T23:12:11.020 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-04-12T23:12:11.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 249953 10.43 MB/sec execute 452 sec latency 80.590 ms 2024-04-12T23:12:11.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:11 smithi144 ceph-mon[67032]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 404 op/s; 213/1629 objects degraded (13.076%) 2024-04-12T23:12:11.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:11 smithi144 ceph-mon[67032]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:11.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:11 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/4032805217' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:12:11.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:11.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:11 smithi181 ceph-mon[52974]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 404 op/s; 213/1629 objects degraded (13.076%) 2024-04-12T23:12:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:11 smithi181 ceph-mon[52974]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:11.849 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:11 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/4032805217' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:12:11.849 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:11.849 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:12.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 250202 10.41 MB/sec execute 453 sec latency 55.621 ms 2024-04-12T23:12:13.206 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:13 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:12:13.031+0000 7f36a5a3e7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-12T23:12:13.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 250432 10.40 MB/sec execute 454 sec latency 60.852 ms 2024-04-12T23:12:13.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:13 smithi144 ceph-mon[67032]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 217/1677 objects degraded (12.940%) 2024-04-12T23:12:13.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:13.749 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:13 smithi181 ceph-mon[52974]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 217/1677 objects degraded (12.940%) 2024-04-12T23:12:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:14.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 250648 10.39 MB/sec execute 455 sec latency 80.011 ms 2024-04-12T23:12:14.816 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:14 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:12:14.809+0000 7f36a5a3e7c0 -1 osd.3 109 log_to_monitors true 2024-04-12T23:12:15.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 250930 10.39 MB/sec execute 456 sec latency 71.462 ms 2024-04-12T23:12:15.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:15 smithi144 ceph-mon[67032]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 219/1683 objects degraded (13.012%) 2024-04-12T23:12:15.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:15 smithi144 ceph-mon[67032]: from='osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T23:12:15.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:15.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:15.580 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:15 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 219/1683 objects degraded (13.012%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:15.580 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:12:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:12:15.568+0000 7f3696e96700 -1 osd.3 109 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:12:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:15 smithi181 ceph-mon[52974]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 219/1683 objects degraded (13.012%) 2024-04-12T23:12:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:15 smithi181 ceph-mon[52974]: from='osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-12T23:12:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:15 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 219/1683 objects degraded (13.012%), 27 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:16.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 251234 10.39 MB/sec execute 457 sec latency 64.471 ms 2024-04-12T23:12:16.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:16 smithi144 ceph-mon[67032]: from='osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T23:12:16.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:16 smithi144 ceph-mon[67032]: osdmap e112: 8 total, 7 up, 8 in 2024-04-12T23:12:16.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:16 smithi144 ceph-mon[67032]: from='osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:12:16.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:16 smithi181 ceph-mon[52974]: from='osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-12T23:12:16.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:16 smithi181 ceph-mon[52974]: osdmap e112: 8 total, 7 up, 8 in 2024-04-12T23:12:16.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:16 smithi181 ceph-mon[52974]: from='osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi144", "root=default"]}]: dispatch 2024-04-12T23:12:17.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 251264 10.36 MB/sec execute 458 sec latency 924.268 ms 2024-04-12T23:12:17.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:17 smithi144 ceph-mon[67032]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 443 op/s; 204/1641 objects degraded (12.431%) 2024-04-12T23:12:17.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:17 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:12:17.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:17 smithi144 ceph-mon[67032]: osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536] boot 2024-04-12T23:12:17.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:17 smithi144 ceph-mon[67032]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T23:12:17.695 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:12:17.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:17 smithi181 ceph-mon[52974]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 443 op/s; 204/1641 objects degraded (12.431%) 2024-04-12T23:12:17.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:17 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:12:17.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:17 smithi181 ceph-mon[52974]: osd.3 [v2:172.21.15.144:6826/2691218536,v1:172.21.15.144:6827/2691218536] boot 2024-04-12T23:12:17.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:17 smithi181 ceph-mon[52974]: osdmap e113: 8 total, 8 up, 8 in 2024-04-12T23:12:17.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-12T23:12:18.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 251264 10.34 MB/sec execute 459 sec latency 1924.300 ms 2024-04-12T23:12:18.801 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:18 smithi144 ceph-mon[67032]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T23:12:18.801 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:18 smithi144 ceph-mon[67032]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T23:12:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:18 smithi181 ceph-mon[52974]: osdmap e114: 8 total, 8 up, 8 in 2024-04-12T23:12:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:18 smithi181 ceph-mon[52974]: osdmap e115: 8 total, 8 up, 8 in 2024-04-12T23:12:19.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 251492 10.32 MB/sec execute 460 sec latency 2148.871 ms 2024-04-12T23:12:19.725 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:19 smithi144 ceph-mon[67032]: pgmap v187: 97 pgs: 13 peering, 8 active+undersized, 20 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.3 MiB/s wr, 310 op/s; 145/1554 objects degraded (9.331%) 2024-04-12T23:12:19.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:19 smithi181 ceph-mon[52974]: pgmap v187: 97 pgs: 13 peering, 8 active+undersized, 20 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.3 MiB/s wr, 310 op/s; 145/1554 objects degraded (9.331%) 2024-04-12T23:12:20.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 251538 10.30 MB/sec execute 461 sec latency 908.055 ms 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:20.693 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:20.693 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:20.693 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:20.693 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:20.693 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 145/1554 objects degraded (9.331%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:20.693 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:20 smithi144 ceph-mon[67032]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T23:12:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: osdmap e116: 8 total, 8 up, 8 in 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 145/1554 objects degraded (9.331%), 20 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:20.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:20 smithi181 ceph-mon[52974]: osdmap e117: 8 total, 8 up, 8 in 2024-04-12T23:12:21.433 INFO:teuthology.orchestra.run.smithi144.stdout: 5 251772 10.30 MB/sec execute 462 sec latency 996.857 ms 2024-04-12T23:12:21.708 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:21 smithi144 ceph-mon[67032]: pgmap v190: 97 pgs: 13 peering, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 2.9 MiB/s wr, 458 op/s; 136/1542 objects degraded (8.820%); 33 KiB/s, 8 objects/s recovering 2024-04-12T23:12:21.709 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:21 smithi144 ceph-mon[67032]: pgmap v191: 97 pgs: 13 peering, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.1 MiB/s wr, 486 op/s; 136/1542 objects degraded (8.820%); 35 KiB/s, 9 objects/s recovering 2024-04-12T23:12:21.709 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:21 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:21.709 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:21 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-12T23:12:21.709 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:21 smithi144 ceph-mon[67032]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T23:12:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:21 smithi181 ceph-mon[52974]: pgmap v190: 97 pgs: 13 peering, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 2.9 MiB/s wr, 458 op/s; 136/1542 objects degraded (8.820%); 33 KiB/s, 8 objects/s recovering 2024-04-12T23:12:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:21 smithi181 ceph-mon[52974]: pgmap v191: 97 pgs: 13 peering, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.1 MiB/s wr, 486 op/s; 136/1542 objects degraded (8.820%); 35 KiB/s, 9 objects/s recovering 2024-04-12T23:12:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:21 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:21 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-12T23:12:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:21 smithi181 ceph-mon[52974]: osdmap e118: 8 total, 8 up, 8 in 2024-04-12T23:12:22.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 252183 10.30 MB/sec execute 463 sec latency 1008.846 ms 2024-04-12T23:12:23.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 252573 10.31 MB/sec execute 464 sec latency 935.363 ms 2024-04-12T23:12:23.498 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:23 smithi144 ceph-mon[67032]: pgmap v194: 97 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 MiB/s wr, 268 op/s; 45/1527 objects degraded (2.947%); 3.9 MiB/s, 43 objects/s recovering 2024-04-12T23:12:23.498 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:23 smithi144 ceph-mon[67032]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T23:12:23.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:23 smithi181 ceph-mon[52974]: pgmap v194: 97 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.2 MiB/s wr, 268 op/s; 45/1527 objects degraded (2.947%); 3.9 MiB/s, 43 objects/s recovering 2024-04-12T23:12:23.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:23 smithi181 ceph-mon[52974]: osdmap e119: 8 total, 8 up, 8 in 2024-04-12T23:12:24.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 252944 10.31 MB/sec execute 465 sec latency 57.012 ms 2024-04-12T23:12:25.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 253333 10.31 MB/sec execute 466 sec latency 42.871 ms 2024-04-12T23:12:25.546 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:25 smithi144 ceph-mon[67032]: pgmap v196: 97 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.3 MiB/s wr, 244 op/s; 45/1533 objects degraded (2.935%); 3.5 MiB/s, 36 objects/s recovering 2024-04-12T23:12:25.546 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:25 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 45/1533 objects degraded (2.935%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:25 smithi181 ceph-mon[52974]: pgmap v196: 97 pgs: 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.3 MiB/s wr, 244 op/s; 45/1533 objects degraded (2.935%); 3.5 MiB/s, 36 objects/s recovering 2024-04-12T23:12:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:25 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 45/1533 objects degraded (2.935%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:26.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 253611 10.30 MB/sec execute 467 sec latency 72.339 ms 2024-04-12T23:12:27.086 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:27 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:27.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:27 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:27.087 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:27 smithi144 ceph-mon[67032]: pgmap v197: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.2 MiB/s wr, 219 op/s; 2.5 MiB/s, 38 objects/s recovering 2024-04-12T23:12:27.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 253942 10.30 MB/sec execute 468 sec latency 57.344 ms 2024-04-12T23:12:27.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:27 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:27.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:27 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:27.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:27 smithi181 ceph-mon[52974]: pgmap v197: 97 pgs: 97 active+clean; 331 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.2 MiB/s wr, 219 op/s; 2.5 MiB/s, 38 objects/s recovering 2024-04-12T23:12:28.123 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:28 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1533 objects degraded (2.935%), 2 pgs degraded) 2024-04-12T23:12:28.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 254257 10.29 MB/sec execute 469 sec latency 90.911 ms 2024-04-12T23:12:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:28 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/1533 objects degraded (2.935%), 2 pgs degraded) 2024-04-12T23:12:29.179 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:29 smithi144 ceph-mon[67032]: pgmap v198: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 346 op/s; 1.9 MiB/s, 29 objects/s recovering 2024-04-12T23:12:29.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 254533 10.28 MB/sec execute 470 sec latency 68.067 ms 2024-04-12T23:12:29.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:29 smithi181 ceph-mon[52974]: pgmap v198: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 346 op/s; 1.9 MiB/s, 29 objects/s recovering 2024-04-12T23:12:30.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 254815 10.28 MB/sec execute 471 sec latency 62.959 ms 2024-04-12T23:12:31.300 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:31 smithi144 ceph-mon[67032]: pgmap v199: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.7 MiB/s wr, 442 op/s; 1.6 MiB/s, 16 objects/s recovering 2024-04-12T23:12:31.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 255073 10.27 MB/sec execute 472 sec latency 55.852 ms 2024-04-12T23:12:31.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:31 smithi181 ceph-mon[52974]: pgmap v199: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.7 MiB/s wr, 442 op/s; 1.6 MiB/s, 16 objects/s recovering 2024-04-12T23:12:32.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 255471 10.27 MB/sec execute 473 sec latency 52.609 ms 2024-04-12T23:12:33.338 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:33 smithi144 ceph-mon[67032]: pgmap v200: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 496 op/s; 80 KiB/s, 8 objects/s recovering 2024-04-12T23:12:33.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 255800 10.27 MB/sec execute 474 sec latency 69.896 ms 2024-04-12T23:12:33.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:33 smithi181 ceph-mon[52974]: pgmap v200: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.2 MiB/s wr, 496 op/s; 80 KiB/s, 8 objects/s recovering 2024-04-12T23:12:34.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 256078 10.27 MB/sec execute 475 sec latency 44.606 ms 2024-04-12T23:12:35.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 256407 10.27 MB/sec execute 476 sec latency 52.869 ms 2024-04-12T23:12:35.435 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:35 smithi144 ceph-mon[67032]: pgmap v201: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 430 op/s; 68 KiB/s, 7 objects/s recovering 2024-04-12T23:12:35.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:35 smithi181 ceph-mon[52974]: pgmap v201: 97 pgs: 97 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 430 op/s; 68 KiB/s, 7 objects/s recovering 2024-04-12T23:12:36.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:36.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:36 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:36.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:36 smithi144 ceph-mon[67032]: Upgrade: osd.4 is safe to restart 2024-04-12T23:12:36.218 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:36.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 256827 10.27 MB/sec execute 477 sec latency 33.001 ms 2024-04-12T23:12:36.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:36.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:36 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-12T23:12:36.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:36 smithi181 ceph-mon[52974]: Upgrade: osd.4 is safe to restart 2024-04-12T23:12:36.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:37.235 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:37 smithi144 ceph-mon[67032]: pgmap v202: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 440 op/s; 67 KiB/s, 7 objects/s recovering 2024-04-12T23:12:37.235 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:37.235 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T23:12:37.235 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:12:37.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 257163 10.26 MB/sec execute 478 sec latency 41.375 ms 2024-04-12T23:12:37.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:37 smithi181 ceph-mon[52974]: pgmap v202: 97 pgs: 97 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 440 op/s; 67 KiB/s, 7 objects/s recovering 2024-04-12T23:12:37.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:37.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-12T23:12:37.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:12:38.245 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:38 smithi144 ceph-mon[67032]: Upgrade: Updating osd.4 2024-04-12T23:12:38.245 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:38 smithi144 ceph-mon[67032]: Deploying daemon osd.4 on smithi181 2024-04-12T23:12:38.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 257575 10.26 MB/sec execute 479 sec latency 47.160 ms 2024-04-12T23:12:38.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:38 smithi181 ceph-mon[52974]: Upgrade: Updating osd.4 2024-04-12T23:12:38.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:38 smithi181 ceph-mon[52974]: Deploying daemon osd.4 on smithi181 2024-04-12T23:12:39.264 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:39 smithi144 ceph-mon[67032]: pgmap v203: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 524 op/s 2024-04-12T23:12:39.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 258116 10.27 MB/sec execute 480 sec latency 34.049 ms 2024-04-12T23:12:39.485 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:39 smithi181 ceph-mon[52974]: pgmap v203: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 524 op/s 2024-04-12T23:12:40.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 258533 10.27 MB/sec execute 481 sec latency 57.695 ms 2024-04-12T23:12:41.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:41 smithi181 ceph-mon[52974]: pgmap v204: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.0 MiB/s wr, 528 op/s 2024-04-12T23:12:41.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:41.317 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:41.317 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:41 smithi181 systemd[1]: Stopping Ceph osd.4 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:12:41.372 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:41 smithi144 ceph-mon[67032]: pgmap v204: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.0 MiB/s wr, 528 op/s 2024-04-12T23:12:41.372 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:41.372 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:41.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 258861 10.27 MB/sec execute 482 sec latency 91.400 ms 2024-04-12T23:12:41.496 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:12:41.697 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:41 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[31157]: 2024-04-12T23:12:41.467+0000 7fef96f71700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:12:41.697 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:41 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[31157]: 2024-04-12T23:12:41.467+0000 7fef96f71700 -1 osd.4 119 *** Got signal Terminated *** 2024-04-12T23:12:41.697 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:41 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[31157]: 2024-04-12T23:12:41.467+0000 7fef96f71700 -1 osd.4 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:12:42.397 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (7m) 29s ago 16m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (3m) 29s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (3m) 3m ago 15m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 29s ago 16m - - 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (10m) 29s ago 10m 38.5M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (10m) 3m ago 10m 14.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (6m) 29s ago 17m 509M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (5m) 29s ago 17m 51.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (4m) 3m ago 15m 35.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (10m) 29s ago 10m 273M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (10m) 3m ago 10m 88.4M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:12:42.398 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 29s ago 16m 9000k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (6m) 3m ago 15m 20.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (3m) 29s ago 14m 337M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (2m) 29s ago 14m 291M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (85s) 29s ago 13m 228M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (38s) 29s ago 13m 12.7M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (13m) 3m ago 13m 556M 2704M 17.2.0 e1d6a67b021e 7bdb2a2de541 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (12m) 3m ago 12m 436M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (12m) 3m ago 12m 339M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (11m) 3m ago 11m 512M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:12:42.399 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (5m) 29s ago 15m 62.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:12:42.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:42 smithi144 ceph-mon[67032]: osd.4 marked itself down and dead 2024-04-12T23:12:42.405 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:42 smithi144 ceph-mon[67032]: from='client.44287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:42.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 259184 10.27 MB/sec execute 483 sec latency 208.256 ms 2024-04-12T23:12:42.521 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:42 smithi181 ceph-mon[52974]: osd.4 marked itself down and dead 2024-04-12T23:12:42.521 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:42 smithi181 ceph-mon[52974]: from='client.44287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 4 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:42.842 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:12:42.843 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:12:43.140 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:42 smithi181 podman[57456]: 2024-04-12 23:12:42.872554575 +0000 UTC m=+1.609620944 container died 7bdb2a2de5417b32d9ac58ea2364cb454ceca7e31334c2281662380338c38f2e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.buildah.version=1.19.8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8) 2024-04-12T23:12:43.179 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:12:43.179 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "10/23 daemons upgraded", 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:12:43.180 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:12:43.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 259258 10.25 MB/sec execute 484 sec latency 1035.151 ms 2024-04-12T23:12:43.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:43 smithi144 ceph-mon[67032]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:43.442 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:43 smithi144 ceph-mon[67032]: pgmap v205: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.9 MiB/s wr, 616 op/s 2024-04-12T23:12:43.443 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:43 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:12:43.443 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:43 smithi144 ceph-mon[67032]: osdmap e120: 8 total, 7 up, 8 in 2024-04-12T23:12:43.443 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:43 smithi144 ceph-mon[67032]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:43.443 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:43 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3864224439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:43.447 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:43 smithi181 podman[57456]: 2024-04-12 23:12:43.242178442 +0000 UTC m=+1.979244820 container cleanup 7bdb2a2de5417b32d9ac58ea2364cb454ceca7e31334c2281662380338c38f2e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, ceph=True, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, version=8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-12T23:12:43.447 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:43 smithi181 bash[57456]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4 2024-04-12T23:12:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:43 smithi181 ceph-mon[52974]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:43 smithi181 ceph-mon[52974]: pgmap v205: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.9 MiB/s wr, 616 op/s 2024-04-12T23:12:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:43 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:12:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:43 smithi181 ceph-mon[52974]: osdmap e120: 8 total, 7 up, 8 in 2024-04-12T23:12:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:43 smithi181 ceph-mon[52974]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:43.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:43 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3864224439' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:43.712 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-04-12T23:12:43.712 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:12:43.712 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:12:43.712 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] OSD_DOWN: 1 osds down 2024-04-12T23:12:43.712 INFO:teuthology.orchestra.run.smithi144.stdout: osd.4 (root=default,host=smithi181) is down 2024-04-12T23:12:44.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:43 smithi181 podman[57485]: 2024-04-12 23:12:43.752697581 +0000 UTC m=+0.883854141 container remove 7bdb2a2de5417b32d9ac58ea2364cb454ceca7e31334c2281662380338c38f2e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, version=8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.buildah.version=1.19.8, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-12T23:12:44.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 259557 10.25 MB/sec execute 485 sec latency 47.010 ms 2024-04-12T23:12:44.450 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:44 smithi144 ceph-mon[67032]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:44.450 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:44 smithi144 ceph-mon[67032]: osdmap e121: 8 total, 7 up, 8 in 2024-04-12T23:12:44.450 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:44 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1382060428' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:12:44.511 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:44 smithi181 ceph-mon[52974]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:12:44.511 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:44 smithi181 ceph-mon[52974]: osdmap e121: 8 total, 7 up, 8 in 2024-04-12T23:12:44.511 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:44 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1382060428' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:12:44.512 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:44 smithi181 podman[57573]: 2024-04-12 23:12:44.130642844 +0000 UTC m=+0.018967185 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:12:44.512 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:44 smithi181 podman[57573]: 2024-04-12 23:12:44.270328174 +0000 UTC m=+0.158652513 container create 800aa40be27d279b470aff168e3520e10d00e6f0bbf914d75b88c928f7c833f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-12T23:12:44.512 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:44 smithi181 podman[57573]: 2024-04-12 23:12:44.524219098 +0000 UTC m=+0.412543448 container init 800aa40be27d279b470aff168e3520e10d00e6f0bbf914d75b88c928f7c833f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-b277feb) 2024-04-12T23:12:44.880 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:44 smithi181 podman[57573]: 2024-04-12 23:12:44.529203805 +0000 UTC m=+0.417528149 container start 800aa40be27d279b470aff168e3520e10d00e6f0bbf914d75b88c928f7c833f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-12T23:12:44.880 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:44 smithi181 podman[57573]: 2024-04-12 23:12:44.631916227 +0000 UTC m=+0.520240580 container attach 800aa40be27d279b470aff168e3520e10d00e6f0bbf914d75b88c928f7c833f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-12T23:12:45.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:44 smithi181 podman[57573]: 2024-04-12 23:12:44.892794878 +0000 UTC m=+0.781119222 container died 800aa40be27d279b470aff168e3520e10d00e6f0bbf914d75b88c928f7c833f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-12T23:12:45.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 259842 10.25 MB/sec execute 486 sec latency 52.851 ms 2024-04-12T23:12:45.494 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:45 smithi144 ceph-mon[67032]: pgmap v208: 97 pgs: 3 active+undersized, 4 active+undersized+degraded, 12 stale+active+clean, 78 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.1 MiB/s wr, 650 op/s; 31/1668 objects degraded (1.859%) 2024-04-12T23:12:45.494 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:45 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 31/1668 objects degraded (1.859%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:45.524 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:45 smithi181 ceph-mon[52974]: pgmap v208: 97 pgs: 3 active+undersized, 4 active+undersized+degraded, 12 stale+active+clean, 78 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.1 MiB/s wr, 650 op/s; 31/1668 objects degraded (1.859%) 2024-04-12T23:12:45.524 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:45 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 31/1668 objects degraded (1.859%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:45.948 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:45 smithi181 podman[57598]: 2024-04-12 23:12:45.536926889 +0000 UTC m=+0.634592879 container remove 800aa40be27d279b470aff168e3520e10d00e6f0bbf914d75b88c928f7c833f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-04-12T23:12:45.948 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:45 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-12T23:12:45.948 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:45 smithi181 systemd[1]: Stopped Ceph osd.4 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:12:45.948 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:45 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.4.service: Consumed 4min 49.480s CPU time. 2024-04-12T23:12:45.948 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:45 smithi181 systemd[1]: Starting Ceph osd.4 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:12:46.270 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 podman[57704]: 2024-04-12 23:12:45.912851945 +0000 UTC m=+0.020059960 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:12:46.270 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 podman[57704]: 2024-04-12 23:12:46.046455919 +0000 UTC m=+0.153663935 container create 963864e422be2bd40f0dd3d1f85dec2e367399a3b1c1ef6d4a5ed2bb32486888 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, GIT_CLEAN=True, ceph=True) 2024-04-12T23:12:46.271 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 podman[57704]: 2024-04-12 23:12:46.282799519 +0000 UTC m=+0.390007551 container init 963864e422be2bd40f0dd3d1f85dec2e367399a3b1c1ef6d4a5ed2bb32486888 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-12T23:12:46.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 260111 10.24 MB/sec execute 487 sec latency 78.254 ms 2024-04-12T23:12:46.697 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 podman[57704]: 2024-04-12 23:12:46.286628772 +0000 UTC m=+0.393836795 container start 963864e422be2bd40f0dd3d1f85dec2e367399a3b1c1ef6d4a5ed2bb32486888 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:12:46.697 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 podman[57704]: 2024-04-12 23:12:46.382882282 +0000 UTC m=+0.490090303 container attach 963864e422be2bd40f0dd3d1f85dec2e367399a3b1c1ef6d4a5ed2bb32486888 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:12:47.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T23:12:47.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T23:12:47.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T23:12:47.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 bash[57704]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate[57718]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-12T23:12:47.198 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:46 smithi181 podman[57704]: 2024-04-12 23:12:46.833592929 +0000 UTC m=+0.940800944 container died 963864e422be2bd40f0dd3d1f85dec2e367399a3b1c1ef6d4a5ed2bb32486888 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b277feb) 2024-04-12T23:12:47.317 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:47 smithi144 ceph-mon[67032]: pgmap v209: 97 pgs: 8 active+undersized, 12 active+undersized+degraded, 7 stale+active+clean, 70 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.2 MiB/s wr, 515 op/s; 105/1626 objects degraded (6.458%) 2024-04-12T23:12:47.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 260384 10.23 MB/sec execute 488 sec latency 63.490 ms 2024-04-12T23:12:47.634 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:47 smithi181 ceph-mon[52974]: pgmap v209: 97 pgs: 8 active+undersized, 12 active+undersized+degraded, 7 stale+active+clean, 70 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.2 MiB/s wr, 515 op/s; 105/1626 objects degraded (6.458%) 2024-04-12T23:12:47.922 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:47 smithi181 podman[57850]: 2024-04-12 23:12:47.646993148 +0000 UTC m=+0.804131131 container remove 963864e422be2bd40f0dd3d1f85dec2e367399a3b1c1ef6d4a5ed2bb32486888 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-12T23:12:47.922 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:47 smithi181 podman[57908]: 2024-04-12 23:12:47.77723202 +0000 UTC m=+0.023180575 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:12:48.196 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:47 smithi181 podman[57908]: 2024-04-12 23:12:47.934898818 +0000 UTC m=+0.180847363 container create d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-12T23:12:48.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:48 smithi181 podman[57908]: 2024-04-12 23:12:48.209405027 +0000 UTC m=+0.455353573 container init d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-04-12T23:12:48.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 260593 10.22 MB/sec execute 489 sec latency 92.747 ms 2024-04-12T23:12:48.447 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:48 smithi181 podman[57908]: 2024-04-12 23:12:48.212998253 +0000 UTC m=+0.458946803 container start d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-12T23:12:48.447 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:48 smithi181 ceph-osd[57927]: -- 172.21.15.181:0/2238232775 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ea026e7680 con 0x55ea026b0c00 2024-04-12T23:12:48.447 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:48 smithi181 bash[57908]: d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 2024-04-12T23:12:48.447 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:48 smithi181 systemd[1]: Started Ceph osd.4 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:12:49.197 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:48 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:12:48.752+0000 7f12aae387c0 -1 Falling back to public interface 2024-04-12T23:12:49.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 260847 10.22 MB/sec execute 490 sec latency 41.030 ms 2024-04-12T23:12:49.595 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:49 smithi144 ceph-mon[67032]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 254/1605 objects degraded (15.826%) 2024-04-12T23:12:49.596 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:49 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:49.596 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:49 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:49.596 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:49 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:49 smithi181 ceph-mon[52974]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 254/1605 objects degraded (15.826%) 2024-04-12T23:12:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:49 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:49 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:49 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:50.434 INFO:teuthology.orchestra.run.smithi144.stdout: 5 261195 10.21 MB/sec execute 491 sec latency 60.119 ms 2024-04-12T23:12:50.497 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:50 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 260/1620 objects degraded (16.049%), 28 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:50.608 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:50 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 260/1620 objects degraded (16.049%), 28 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:51.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 261526 10.21 MB/sec execute 492 sec latency 61.794 ms 2024-04-12T23:12:51.622 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:51 smithi144 ceph-mon[67032]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.0 MiB/s wr, 253 op/s; 260/1620 objects degraded (16.049%) 2024-04-12T23:12:51.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:51 smithi181 ceph-mon[52974]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.0 MiB/s wr, 253 op/s; 260/1620 objects degraded (16.049%) 2024-04-12T23:12:52.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 261789 10.20 MB/sec execute 493 sec latency 37.609 ms 2024-04-12T23:12:53.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 262058 10.20 MB/sec execute 494 sec latency 42.400 ms 2024-04-12T23:12:53.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:53 smithi144 ceph-mon[67032]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 399 op/s; 266/1653 objects degraded (16.092%) 2024-04-12T23:12:53.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:53.673 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:53.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:53 smithi181 ceph-mon[52974]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.9 MiB/s wr, 399 op/s; 266/1653 objects degraded (16.092%) 2024-04-12T23:12:53.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:53 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:53.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:53 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:54.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 262288 10.19 MB/sec execute 495 sec latency 84.787 ms 2024-04-12T23:12:55.104 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:55.105 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:55.105 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:55 smithi181 ceph-mon[52974]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 371 op/s; 266/1635 objects degraded (16.269%) 2024-04-12T23:12:55.221 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:55.222 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:55.222 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:55 smithi144 ceph-mon[67032]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 371 op/s; 266/1635 objects degraded (16.269%) 2024-04-12T23:12:55.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 262596 10.18 MB/sec execute 496 sec latency 78.736 ms 2024-04-12T23:12:56.074 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:12:55.828+0000 7f12aae387c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-12T23:12:56.242 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:56 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 266/1635 objects degraded (16.269%), 28 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:56.329 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:56 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 266/1635 objects degraded (16.269%), 28 pgs degraded (PG_DEGRADED) 2024-04-12T23:12:56.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 262807 10.17 MB/sec execute 497 sec latency 57.844 ms 2024-04-12T23:12:57.284 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:57.284 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:57.285 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:57 smithi144 ceph-mon[67032]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 356 op/s; 259/1617 objects degraded (16.017%) 2024-04-12T23:12:57.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 263028 10.16 MB/sec execute 498 sec latency 57.545 ms 2024-04-12T23:12:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:12:57.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:57 smithi181 ceph-mon[52974]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 356 op/s; 259/1617 objects degraded (16.017%) 2024-04-12T23:12:57.947 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:12:57.483+0000 7f12aae387c0 -1 osd.4 119 log_to_monitors true 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='osd.4 [v2:172.21.15.181:6800/4068302462,v1:172.21.15.181:6801/4068302462]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.311 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:58 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 263330 10.17 MB/sec execute 499 sec latency 90.591 ms 2024-04-12T23:12:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='osd.4 [v2:172.21.15.181:6800/4068302462,v1:172.21.15.181:6801/4068302462]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T23:12:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-12T23:12:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:12:58.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:12:58.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:12:58.947 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:12:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:12:58.591+0000 7f1298288700 -1 osd.4 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 462 op/s; 257/1602 objects degraded (16.042%) 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 414 op/s; 257/1602 objects degraded (16.042%) 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='osd.4 [v2:172.21.15.181:6800/4068302462,v1:172.21.15.181:6801/4068302462]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: osdmap e122: 8 total, 7 up, 8 in 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:59.321 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:59.322 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:59.322 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:59.322 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:12:59.322 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:12:59.322 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-12T23:12:59.322 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:12:59 smithi144 ceph-mon[67032]: from='osd.4 ' entity='osd.4' 2024-04-12T23:12:59.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 263784 10.17 MB/sec execute 500 sec latency 61.272 ms 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.3 MiB/s wr, 462 op/s; 257/1602 objects degraded (16.042%) 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 414 op/s; 257/1602 objects degraded (16.042%) 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='osd.4 [v2:172.21.15.181:6800/4068302462,v1:172.21.15.181:6801/4068302462]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: osdmap e122: 8 total, 7 up, 8 in 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:59.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:12:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:12:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:12:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-04-12T23:12:59.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:12:59 smithi181 ceph-mon[52974]: from='osd.4 ' entity='osd.4' 2024-04-12T23:13:00.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 263930 10.15 MB/sec execute 501 sec latency 853.428 ms 2024-04-12T23:13:00.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:00 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:13:00.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:00 smithi181 ceph-mon[52974]: osd.4 [v2:172.21.15.181:6800/4068302462,v1:172.21.15.181:6801/4068302462] boot 2024-04-12T23:13:00.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:00 smithi181 ceph-mon[52974]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T23:13:00.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:00 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:13:00.511 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:00 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:13:00.511 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:00 smithi144 ceph-mon[67032]: osd.4 [v2:172.21.15.181:6800/4068302462,v1:172.21.15.181:6801/4068302462] boot 2024-04-12T23:13:00.512 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:00 smithi144 ceph-mon[67032]: osdmap e123: 8 total, 8 up, 8 in 2024-04-12T23:13:00.512 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:00 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-12T23:13:01.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 263975 10.14 MB/sec execute 502 sec latency 1852.430 ms 2024-04-12T23:13:01.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:01 smithi181 ceph-mon[52974]: pgmap v219: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.2 MiB/s wr, 319 op/s; 255/1599 objects degraded (15.947%) 2024-04-12T23:13:01.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:01 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 257/1602 objects degraded (16.042%), 28 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:01.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:01 smithi181 ceph-mon[52974]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T23:13:01.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:01 smithi144 ceph-mon[67032]: pgmap v219: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.2 MiB/s wr, 319 op/s; 255/1599 objects degraded (15.947%) 2024-04-12T23:13:01.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:01 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 257/1602 objects degraded (16.042%), 28 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:01.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:01 smithi144 ceph-mon[67032]: osdmap e124: 8 total, 8 up, 8 in 2024-04-12T23:13:02.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 264221 10.13 MB/sec execute 503 sec latency 1900.432 ms 2024-04-12T23:13:02.544 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:02 smithi144 ceph-mon[67032]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T23:13:02.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:02 smithi181 ceph-mon[52974]: osdmap e125: 8 total, 8 up, 8 in 2024-04-12T23:13:03.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 264512 10.13 MB/sec execute 504 sec latency 969.734 ms 2024-04-12T23:13:03.555 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:03 smithi144 ceph-mon[67032]: pgmap v222: 97 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 6 peering, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 67 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 387 op/s; 153/1566 objects degraded (9.770%); 0 B/s, 20 objects/s recovering 2024-04-12T23:13:03.556 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:03 smithi144 ceph-mon[67032]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T23:13:03.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:03 smithi181 ceph-mon[52974]: pgmap v222: 97 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 6 peering, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded, 67 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 387 op/s; 153/1566 objects degraded (9.770%); 0 B/s, 20 objects/s recovering 2024-04-12T23:13:03.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:03 smithi181 ceph-mon[52974]: osdmap e126: 8 total, 8 up, 8 in 2024-04-12T23:13:04.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 264857 10.12 MB/sec execute 505 sec latency 983.176 ms 2024-04-12T23:13:04.583 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:04 smithi144 ceph-mon[67032]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T23:13:04.584 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:04 smithi144 ceph-mon[67032]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T23:13:04.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:04 smithi181 ceph-mon[52974]: osdmap e127: 8 total, 8 up, 8 in 2024-04-12T23:13:04.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:04 smithi181 ceph-mon[52974]: osdmap e128: 8 total, 8 up, 8 in 2024-04-12T23:13:05.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 265195 10.12 MB/sec execute 506 sec latency 278.877 ms 2024-04-12T23:13:05.600 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:05 smithi144 ceph-mon[67032]: pgmap v225: 97 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 8 peering, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 526 op/s; 69/1593 objects degraded (4.331%); 200 KiB/s, 61 keys/s, 30 objects/s recovering 2024-04-12T23:13:05.601 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:05 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 69/1593 objects degraded (4.331%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:05.601 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:05 smithi144 ceph-mon[67032]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T23:13:05.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:05 smithi181 ceph-mon[52974]: pgmap v225: 97 pgs: 2 active+recovery_wait+degraded, 1 remapped+peering, 8 peering, 2 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 526 op/s; 69/1593 objects degraded (4.331%); 200 KiB/s, 61 keys/s, 30 objects/s recovering 2024-04-12T23:13:05.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:05 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 69/1593 objects degraded (4.331%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:05.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:05 smithi181 ceph-mon[52974]: osdmap e129: 8 total, 8 up, 8 in 2024-04-12T23:13:06.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 265510 10.12 MB/sec execute 507 sec latency 572.258 ms 2024-04-12T23:13:07.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 265840 10.12 MB/sec execute 508 sec latency 580.812 ms 2024-04-12T23:13:07.655 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:07 smithi144 ceph-mon[67032]: pgmap v228: 97 pgs: 2 active+recovery_wait+degraded, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 259 op/s; 69/1581 objects degraded (4.364%); 803 KiB/s, 61 keys/s, 29 objects/s recovering 2024-04-12T23:13:07.655 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:07 smithi144 ceph-mon[67032]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T23:13:07.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:07 smithi181 ceph-mon[52974]: pgmap v228: 97 pgs: 2 active+recovery_wait+degraded, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 259 op/s; 69/1581 objects degraded (4.364%); 803 KiB/s, 61 keys/s, 29 objects/s recovering 2024-04-12T23:13:07.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:07 smithi181 ceph-mon[52974]: osdmap e130: 8 total, 8 up, 8 in 2024-04-12T23:13:08.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 266207 10.12 MB/sec execute 509 sec latency 63.711 ms 2024-04-12T23:13:08.672 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:08 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/1581 objects degraded (4.364%), 4 pgs degraded) 2024-04-12T23:13:08.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:08 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 69/1581 objects degraded (4.364%), 4 pgs degraded) 2024-04-12T23:13:09.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 266514 10.11 MB/sec execute 510 sec latency 62.551 ms 2024-04-12T23:13:09.692 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:09 smithi144 ceph-mon[67032]: pgmap v230: 97 pgs: 2 peering, 95 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.1 MiB/s wr, 274 op/s; 521 KiB/s, 31 objects/s recovering 2024-04-12T23:13:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:09 smithi181 ceph-mon[52974]: pgmap v230: 97 pgs: 2 peering, 95 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.1 MiB/s wr, 274 op/s; 521 KiB/s, 31 objects/s recovering 2024-04-12T23:13:10.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 266809 10.11 MB/sec execute 511 sec latency 41.677 ms 2024-04-12T23:13:11.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 267125 10.10 MB/sec execute 512 sec latency 52.085 ms 2024-04-12T23:13:11.728 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:11 smithi144 ceph-mon[67032]: pgmap v231: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.6 MiB/s wr, 349 op/s; 536 KiB/s, 31 objects/s recovering 2024-04-12T23:13:11.729 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:11.729 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:11.749 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:11 smithi181 ceph-mon[52974]: pgmap v231: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.6 MiB/s wr, 349 op/s; 536 KiB/s, 31 objects/s recovering 2024-04-12T23:13:11.749 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:11.749 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:12.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 267462 10.10 MB/sec execute 513 sec latency 49.067 ms 2024-04-12T23:13:13.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 267804 10.11 MB/sec execute 514 sec latency 61.561 ms 2024-04-12T23:13:13.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:13 smithi144 ceph-mon[67032]: pgmap v232: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.2 MiB/s wr, 338 op/s; 422 KiB/s, 25 objects/s recovering 2024-04-12T23:13:13.587 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:13:13.941 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:13 smithi181 ceph-mon[52974]: pgmap v232: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.2 MiB/s wr, 338 op/s; 422 KiB/s, 25 objects/s recovering 2024-04-12T23:13:13.941 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:13:14.188 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:13:14.435 INFO:teuthology.orchestra.run.smithi144.stdout: 5 268026 10.10 MB/sec execute 515 sec latency 57.923 ms 2024-04-12T23:13:14.606 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:14 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:13:14.607 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:14 smithi144 ceph-mon[67032]: Upgrade: osd.5 is safe to restart 2024-04-12T23:13:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:14 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-12T23:13:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:14 smithi181 ceph-mon[52974]: Upgrade: osd.5 is safe to restart 2024-04-12T23:13:15.205 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:13:15.205 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (8m) 61s ago 16m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:13:15.205 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (4m) 61s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (4m) 22s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 61s ago 16m - - 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (11m) 61s ago 11m 38.5M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (11m) 22s ago 11m 15.1M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (6m) 61s ago 18m 509M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (6m) 22s ago 16m 452M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (5m) 61s ago 18m 51.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (5m) 22s ago 16m 39.3M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (10m) 61s ago 10m 273M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (10m) 22s ago 10m 88.7M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (7m) 61s ago 16m 9000k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 22s ago 15m 16.8M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:13:15.206 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (3m) 61s ago 15m 337M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (2m) 61s ago 14m 291M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (118s) 61s ago 14m 228M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (70s) 61s ago 14m 12.7M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (27s) 22s ago 13m 11.9M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (13m) 22s ago 13m 629M 2704M 17.2.0 e1d6a67b021e a7d6309b8f47 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (12m) 22s ago 12m 546M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (12m) 22s ago 12m 556M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:13:15.207 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (6m) 61s ago 15m 62.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:13:15.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 268399 10.09 MB/sec execute 516 sec latency 61.540 ms 2024-04-12T23:13:15.620 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:15 smithi144 ceph-mon[67032]: pgmap v233: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 514 op/s; 205 KiB/s, 15 objects/s recovering 2024-04-12T23:13:15.620 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:15 smithi144 ceph-mon[67032]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:15.620 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:15.620 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T23:13:15.620 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:13:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:15 smithi181 ceph-mon[52974]: pgmap v233: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 514 op/s; 205 KiB/s, 15 objects/s recovering 2024-04-12T23:13:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:15 smithi181 ceph-mon[52974]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-12T23:13:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:13:15.725 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 5 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 9 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:13:15.726 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:13:16.227 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:13:16.227 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "11/23 daemons upgraded", 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:13:16.228 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:13:16.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 268670 10.08 MB/sec execute 517 sec latency 45.643 ms 2024-04-12T23:13:16.627 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:16 smithi144 ceph-mon[67032]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:16.628 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:16 smithi144 ceph-mon[67032]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:16.628 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:16 smithi144 ceph-mon[67032]: Upgrade: Updating osd.5 2024-04-12T23:13:16.628 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:16 smithi144 ceph-mon[67032]: Deploying daemon osd.5 on smithi181 2024-04-12T23:13:16.628 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:16 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2014337770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:16.628 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:16 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:16 smithi181 ceph-mon[52974]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:16 smithi181 ceph-mon[52974]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:16 smithi181 ceph-mon[52974]: Upgrade: Updating osd.5 2024-04-12T23:13:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:16 smithi181 ceph-mon[52974]: Deploying daemon osd.5 on smithi181 2024-04-12T23:13:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:16 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2014337770' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:16 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:16.875 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:13:16.876 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:13:16.876 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:13:17.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 268977 10.08 MB/sec execute 518 sec latency 43.503 ms 2024-04-12T23:13:17.634 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:17 smithi144 ceph-mon[67032]: pgmap v234: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 439 op/s; 81 KiB/s, 11 objects/s recovering 2024-04-12T23:13:17.635 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:17 smithi144 ceph-mon[67032]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:17.635 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:17 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1117232909' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:13:17.694 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:17 smithi181 ceph-mon[52974]: pgmap v234: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 439 op/s; 81 KiB/s, 11 objects/s recovering 2024-04-12T23:13:17.694 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:17 smithi181 ceph-mon[52974]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:17.694 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:17 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1117232909' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:13:18.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 269297 10.09 MB/sec execute 519 sec latency 46.582 ms 2024-04-12T23:13:19.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 269672 10.09 MB/sec execute 520 sec latency 47.919 ms 2024-04-12T23:13:19.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:19 smithi181 systemd[1]: Stopping Ceph osd.5 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:13:19.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:19 smithi144 ceph-mon[67032]: pgmap v235: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.8 MiB/s wr, 519 op/s; 69 KiB/s, 9 objects/s recovering 2024-04-12T23:13:19.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:19 smithi144 ceph-mon[67032]: osd.5 marked itself down and dead 2024-04-12T23:13:19.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:19 smithi181 ceph-mon[52974]: pgmap v235: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.8 MiB/s wr, 519 op/s; 69 KiB/s, 9 objects/s recovering 2024-04-12T23:13:19.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:19 smithi181 ceph-mon[52974]: osd.5 marked itself down and dead 2024-04-12T23:13:19.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:19 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[34118]: 2024-04-12T23:13:19.517+0000 7f37ce53d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:13:19.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:19 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[34118]: 2024-04-12T23:13:19.517+0000 7f37ce53d700 -1 osd.5 130 *** Got signal Terminated *** 2024-04-12T23:13:19.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:19 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[34118]: 2024-04-12T23:13:19.517+0000 7f37ce53d700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:13:20.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 269708 10.07 MB/sec execute 521 sec latency 908.331 ms 2024-04-12T23:13:20.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:20 smithi181 podman[59728]: 2024-04-12 23:13:20.200559836 +0000 UTC m=+0.979369903 container died a7d6309b8f47369c2f7ae2ed18ee9f5ca6ccd1cd14e2c2b9cd5784ea1b76f244 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD) 2024-04-12T23:13:20.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:20 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:13:20.777 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:20 smithi144 ceph-mon[67032]: osdmap e131: 8 total, 7 up, 8 in 2024-04-12T23:13:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:20 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:13:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:20 smithi181 ceph-mon[52974]: osdmap e131: 8 total, 7 up, 8 in 2024-04-12T23:13:20.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:20 smithi181 podman[59728]: 2024-04-12 23:13:20.559120167 +0000 UTC m=+1.337930236 container cleanup a7d6309b8f47369c2f7ae2ed18ee9f5ca6ccd1cd14e2c2b9cd5784ea1b76f244 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, name=centos-stream, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_CLEAN=True, GIT_BRANCH=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-12T23:13:20.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:20 smithi181 bash[59728]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5 2024-04-12T23:13:21.392 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:20 smithi181 podman[59747]: 2024-04-12 23:13:20.985806944 +0000 UTC m=+0.780686856 container remove a7d6309b8f47369c2f7ae2ed18ee9f5ca6ccd1cd14e2c2b9cd5784ea1b76f244 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, architecture=x86_64, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-type=git, version=8, GIT_BRANCH=HEAD, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T23:13:21.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 269986 10.06 MB/sec execute 522 sec latency 1057.391 ms 2024-04-12T23:13:21.565 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:21 smithi144 ceph-mon[67032]: pgmap v237: 97 pgs: 12 stale+active+clean, 85 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.4 MiB/s wr, 455 op/s 2024-04-12T23:13:21.565 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:21 smithi144 ceph-mon[67032]: osdmap e132: 8 total, 7 up, 8 in 2024-04-12T23:13:21.669 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:21 smithi181 ceph-mon[52974]: pgmap v237: 97 pgs: 12 stale+active+clean, 85 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.4 MiB/s wr, 455 op/s 2024-04-12T23:13:21.669 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:21 smithi181 ceph-mon[52974]: osdmap e132: 8 total, 7 up, 8 in 2024-04-12T23:13:21.669 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:21 smithi181 podman[59838]: 2024-04-12 23:13:21.309195413 +0000 UTC m=+0.021135702 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:13:21.669 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:21 smithi181 podman[59838]: 2024-04-12 23:13:21.437015699 +0000 UTC m=+0.148955993 container create fdd2bc0321681ab2b3a19f5be7e5bce4aa5ed2332f6be4380a3f2bb1fa39be2c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:13:21.669 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:21 smithi181 podman[59838]: 2024-04-12 23:13:21.681621407 +0000 UTC m=+0.393561699 container init fdd2bc0321681ab2b3a19f5be7e5bce4aa5ed2332f6be4380a3f2bb1fa39be2c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-12T23:13:21.919 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:21 smithi181 podman[59838]: 2024-04-12 23:13:21.686746828 +0000 UTC m=+0.398687120 container start fdd2bc0321681ab2b3a19f5be7e5bce4aa5ed2332f6be4380a3f2bb1fa39be2c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, ceph=True) 2024-04-12T23:13:22.197 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:21 smithi181 podman[59838]: 2024-04-12 23:13:21.931899114 +0000 UTC m=+0.643839407 container attach fdd2bc0321681ab2b3a19f5be7e5bce4aa5ed2332f6be4380a3f2bb1fa39be2c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b277feb, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:13:22.197 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:22 smithi181 podman[59838]: 2024-04-12 23:13:22.055214071 +0000 UTC m=+0.767154363 container died fdd2bc0321681ab2b3a19f5be7e5bce4aa5ed2332f6be4380a3f2bb1fa39be2c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:13:22.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 270311 10.06 MB/sec execute 523 sec latency 37.363 ms 2024-04-12T23:13:22.588 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:22 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 84/1632 objects degraded (5.147%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:22.680 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:22 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 84/1632 objects degraded (5.147%), 11 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:22.680 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:22 smithi181 podman[59865]: 2024-04-12 23:13:22.678621064 +0000 UTC m=+0.617419830 container remove fdd2bc0321681ab2b3a19f5be7e5bce4aa5ed2332f6be4380a3f2bb1fa39be2c (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:13:22.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:22 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-12T23:13:22.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:22 smithi181 systemd[1]: Stopped Ceph osd.5 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:13:22.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:22 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.5.service: Consumed 5min 54.500s CPU time. 2024-04-12T23:13:22.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:22 smithi181 systemd[1]: Starting Ceph osd.5 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:13:23.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 270627 10.05 MB/sec execute 524 sec latency 43.328 ms 2024-04-12T23:13:23.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:23 smithi181 podman[59964]: 2024-04-12 23:13:23.089305233 +0000 UTC m=+0.031247236 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:13:23.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:23 smithi181 podman[59964]: 2024-04-12 23:13:23.221378041 +0000 UTC m=+0.163320030 container create 589b437221577ed08e50e6b82c6e742bfbbaf0d351f943f2bf15d1c603e5b817 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-12T23:13:23.598 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:23 smithi144 ceph-mon[67032]: pgmap v239: 97 pgs: 1 active+undersized, 11 active+undersized+degraded, 4 peering, 7 stale+active+clean, 74 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 415 op/s; 84/1632 objects degraded (5.147%) 2024-04-12T23:13:23.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:23 smithi181 ceph-mon[52974]: pgmap v239: 97 pgs: 1 active+undersized, 11 active+undersized+degraded, 4 peering, 7 stale+active+clean, 74 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.8 MiB/s wr, 415 op/s; 84/1632 objects degraded (5.147%) 2024-04-12T23:13:23.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:23 smithi181 podman[59964]: 2024-04-12 23:13:23.479480084 +0000 UTC m=+0.421422076 container init 589b437221577ed08e50e6b82c6e742bfbbaf0d351f943f2bf15d1c603e5b817 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True) 2024-04-12T23:13:23.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:23 smithi181 podman[59964]: 2024-04-12 23:13:23.483046846 +0000 UTC m=+0.424988840 container start 589b437221577ed08e50e6b82c6e742bfbbaf0d351f943f2bf15d1c603e5b817 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, GIT_CLEAN=True) 2024-04-12T23:13:23.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:23 smithi181 podman[59964]: 2024-04-12 23:13:23.583124925 +0000 UTC m=+0.525066922 container attach 589b437221577ed08e50e6b82c6e742bfbbaf0d351f943f2bf15d1c603e5b817 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409) 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 bash[59964]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-12T23:13:24.341 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-12T23:13:24.342 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-12T23:13:24.342 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-12T23:13:24.342 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-12T23:13:24.342 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate[59977]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-12T23:13:24.342 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 podman[59964]: 2024-04-12 23:13:24.047019444 +0000 UTC m=+0.988961439 container died 589b437221577ed08e50e6b82c6e742bfbbaf0d351f943f2bf15d1c603e5b817 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate, ceph=True, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409) 2024-04-12T23:13:24.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 271011 10.06 MB/sec execute 525 sec latency 65.539 ms 2024-04-12T23:13:24.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 podman[60109]: 2024-04-12 23:13:24.671455776 +0000 UTC m=+0.619170611 container remove 589b437221577ed08e50e6b82c6e742bfbbaf0d351f943f2bf15d1c603e5b817 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:13:24.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:24 smithi181 podman[60169]: 2024-04-12 23:13:24.799056097 +0000 UTC m=+0.020353411 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:13:25.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 271378 10.06 MB/sec execute 526 sec latency 58.901 ms 2024-04-12T23:13:25.571 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:25 smithi181 podman[60169]: 2024-04-12 23:13:25.255946967 +0000 UTC m=+0.477244253 container create d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-12T23:13:25.615 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:25 smithi144 ceph-mon[67032]: pgmap v240: 97 pgs: 6 active+undersized, 21 active+undersized+degraded, 4 peering, 66 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 430 op/s; 159/1638 objects degraded (9.707%) 2024-04-12T23:13:25.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:25 smithi181 podman[60169]: 2024-04-12 23:13:25.584066699 +0000 UTC m=+0.805363996 container init d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-12T23:13:25.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:25 smithi181 podman[60169]: 2024-04-12 23:13:25.587782509 +0000 UTC m=+0.809079800 container start d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-b277feb) 2024-04-12T23:13:25.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:25 smithi181 ceph-osd[60187]: -- 172.21.15.181:0/3454491435 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d2010a1680 con 0x55d20106b000 2024-04-12T23:13:25.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:25 smithi181 bash[60169]: d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 2024-04-12T23:13:25.948 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:25 smithi181 systemd[1]: Started Ceph osd.5 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:13:25.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:25 smithi181 ceph-mon[52974]: pgmap v240: 97 pgs: 6 active+undersized, 21 active+undersized+degraded, 4 peering, 66 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.2 MiB/s wr, 430 op/s; 159/1638 objects degraded (9.707%) 2024-04-12T23:13:26.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 271687 10.05 MB/sec execute 527 sec latency 57.882 ms 2024-04-12T23:13:26.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:13:26.136+0000 7fe0d27307c0 -1 Falling back to public interface 2024-04-12T23:13:26.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:26.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:26.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:13:26.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:26 smithi144 ceph-mon[67032]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.6 MiB/s wr, 396 op/s; 186/1653 objects degraded (11.252%) 2024-04-12T23:13:26.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:26.874 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:27.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:27.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:27.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:13:27.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-mon[52974]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.6 MiB/s wr, 396 op/s; 186/1653 objects degraded (11.252%) 2024-04-12T23:13:27.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:27.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:27.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 272200 10.05 MB/sec execute 528 sec latency 30.761 ms 2024-04-12T23:13:28.402 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:28 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 188/1644 objects degraded (11.436%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:28.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 272700 10.07 MB/sec execute 529 sec latency 50.058 ms 2024-04-12T23:13:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:28 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 188/1644 objects degraded (11.436%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:29.165 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:29 smithi181 ceph-mon[52974]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 514 op/s; 188/1644 objects degraded (11.436%) 2024-04-12T23:13:29.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:29 smithi144 ceph-mon[67032]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 514 op/s; 188/1644 objects degraded (11.436%) 2024-04-12T23:13:29.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 273154 10.07 MB/sec execute 530 sec latency 53.102 ms 2024-04-12T23:13:30.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 273468 10.07 MB/sec execute 531 sec latency 68.757 ms 2024-04-12T23:13:31.219 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:31 smithi144 ceph-mon[67032]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.1 MiB/s wr, 535 op/s; 195/1698 objects degraded (11.484%) 2024-04-12T23:13:31.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:31 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:31.220 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:31 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:31.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 273765 10.06 MB/sec execute 532 sec latency 43.149 ms 2024-04-12T23:13:31.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:31 smithi181 ceph-mon[52974]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.1 MiB/s wr, 535 op/s; 195/1698 objects degraded (11.484%) 2024-04-12T23:13:31.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:31 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:31.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:31 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:32.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 273964 10.05 MB/sec execute 533 sec latency 62.099 ms 2024-04-12T23:13:32.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:32.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:33.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 274346 10.06 MB/sec execute 534 sec latency 41.972 ms 2024-04-12T23:13:33.679 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:33 smithi181 ceph-mon[52974]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.1 MiB/s wr, 526 op/s; 189/1683 objects degraded (11.230%) 2024-04-12T23:13:33.796 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:33 smithi144 ceph-mon[67032]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 5.1 MiB/s wr, 526 op/s; 189/1683 objects degraded (11.230%) 2024-04-12T23:13:34.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 274688 10.05 MB/sec execute 535 sec latency 42.075 ms 2024-04-12T23:13:34.629 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:34 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 183/1647 objects degraded (11.111%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:34.858 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:34 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 183/1647 objects degraded (11.111%), 23 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:35.197 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:13:35.108+0000 7fe0d27307c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-12T23:13:35.436 INFO:teuthology.orchestra.run.smithi144.stdout: 5 275008 10.05 MB/sec execute 536 sec latency 52.265 ms 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 499 op/s; 183/1647 objects degraded (11.111%) 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:35.860 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:35.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:35.861 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:35 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:35.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 499 op/s; 183/1647 objects degraded (11.111%) 2024-04-12T23:13:35.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:35.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:35 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:36.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 275361 10.04 MB/sec execute 537 sec latency 57.550 ms 2024-04-12T23:13:36.617 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:36 smithi144 ceph-mon[67032]: pgmap v246: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 516 op/s; 181/1623 objects degraded (11.152%) 2024-04-12T23:13:36.617 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:36 smithi144 ceph-mon[67032]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 478 op/s; 181/1623 objects degraded (11.152%) 2024-04-12T23:13:36.617 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:36 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:36.617 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:36 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-12T23:13:36.617 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:36 smithi144 ceph-mon[67032]: from='osd.5 [v2:172.21.15.181:6808/1065885728,v1:172.21.15.181:6809/1065885728]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T23:13:36.617 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:36 smithi144 ceph-mon[67032]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T23:13:36.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:13:36.518+0000 7fe0d27307c0 -1 osd.5 130 log_to_monitors true 2024-04-12T23:13:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-mon[52974]: pgmap v246: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 516 op/s; 181/1623 objects degraded (11.152%) 2024-04-12T23:13:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-mon[52974]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 478 op/s; 181/1623 objects degraded (11.152%) 2024-04-12T23:13:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-04-12T23:13:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-mon[52974]: from='osd.5 [v2:172.21.15.181:6808/1065885728,v1:172.21.15.181:6809/1065885728]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T23:13:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:36 smithi181 ceph-mon[52974]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-12T23:13:37.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 275719 10.04 MB/sec execute 538 sec latency 47.828 ms 2024-04-12T23:13:37.697 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:13:37 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:13:37.326+0000 7fe0c3b88700 -1 osd.5 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:13:38.424 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:38 smithi144 ceph-mon[67032]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T23:13:38.425 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:38 smithi144 ceph-mon[67032]: from='osd.5 [v2:172.21.15.181:6808/1065885728,v1:172.21.15.181:6809/1065885728]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:13:38.425 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:38 smithi144 ceph-mon[67032]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 556 op/s; 187/1617 objects degraded (11.565%) 2024-04-12T23:13:38.425 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:38 smithi144 ceph-mon[67032]: osdmap e133: 8 total, 7 up, 8 in 2024-04-12T23:13:38.425 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:38 smithi144 ceph-mon[67032]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:13:38.440 INFO:teuthology.orchestra.run.smithi144.stdout: 5 275996 10.04 MB/sec execute 539 sec latency 59.211 ms 2024-04-12T23:13:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:38 smithi181 ceph-mon[52974]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-12T23:13:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:38 smithi181 ceph-mon[52974]: from='osd.5 [v2:172.21.15.181:6808/1065885728,v1:172.21.15.181:6809/1065885728]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:13:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:38 smithi181 ceph-mon[52974]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 556 op/s; 187/1617 objects degraded (11.565%) 2024-04-12T23:13:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:38 smithi181 ceph-mon[52974]: osdmap e133: 8 total, 7 up, 8 in 2024-04-12T23:13:38.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:38 smithi181 ceph-mon[52974]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:13:39.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 276101 10.03 MB/sec execute 540 sec latency 963.759 ms 2024-04-12T23:13:39.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:39 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:13:39.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:39 smithi144 ceph-mon[67032]: osd.5 [v2:172.21.15.181:6808/1065885728,v1:172.21.15.181:6809/1065885728] boot 2024-04-12T23:13:39.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:39 smithi144 ceph-mon[67032]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T23:13:39.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:39 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:13:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:39 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:13:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:39 smithi181 ceph-mon[52974]: osd.5 [v2:172.21.15.181:6808/1065885728,v1:172.21.15.181:6809/1065885728] boot 2024-04-12T23:13:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:39 smithi181 ceph-mon[52974]: osdmap e134: 8 total, 8 up, 8 in 2024-04-12T23:13:39.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:39 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-12T23:13:40.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 276173 10.02 MB/sec execute 541 sec latency 1853.656 ms 2024-04-12T23:13:40.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:40 smithi144 ceph-mon[67032]: pgmap v251: 97 pgs: 4 peering, 6 active+undersized, 21 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.3 MiB/s wr, 454 op/s; 167/1605 objects degraded (10.405%) 2024-04-12T23:13:40.578 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:40 smithi144 ceph-mon[67032]: osdmap e135: 8 total, 8 up, 8 in 2024-04-12T23:13:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:40 smithi181 ceph-mon[52974]: pgmap v251: 97 pgs: 4 peering, 6 active+undersized, 21 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.3 MiB/s wr, 454 op/s; 167/1605 objects degraded (10.405%) 2024-04-12T23:13:40.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:40 smithi181 ceph-mon[52974]: osdmap e135: 8 total, 8 up, 8 in 2024-04-12T23:13:41.347 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:41 smithi144 ceph-mon[67032]: osdmap e136: 8 total, 8 up, 8 in 2024-04-12T23:13:41.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:41 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 167/1605 objects degraded (10.405%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:41.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:41.348 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:41.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 276411 10.01 MB/sec execute 542 sec latency 98.468 ms 2024-04-12T23:13:41.598 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:41 smithi181 ceph-mon[52974]: osdmap e136: 8 total, 8 up, 8 in 2024-04-12T23:13:41.598 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:41 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 167/1605 objects degraded (10.405%), 21 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:41.598 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:41.598 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:42.385 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:42 smithi144 ceph-mon[67032]: pgmap v254: 97 pgs: 1 remapped+peering, 4 peering, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 70 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 411 op/s; 156/1617 objects degraded (9.647%); 161 KiB/s, 4 objects/s recovering 2024-04-12T23:13:42.386 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:42 smithi144 ceph-mon[67032]: osdmap e137: 8 total, 8 up, 8 in 2024-04-12T23:13:42.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 276745 10.01 MB/sec execute 543 sec latency 1034.853 ms 2024-04-12T23:13:42.613 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:42 smithi181 ceph-mon[52974]: pgmap v254: 97 pgs: 1 remapped+peering, 4 peering, 4 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 70 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 411 op/s; 156/1617 objects degraded (9.647%); 161 KiB/s, 4 objects/s recovering 2024-04-12T23:13:42.613 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:42 smithi181 ceph-mon[52974]: osdmap e137: 8 total, 8 up, 8 in 2024-04-12T23:13:43.401 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:43 smithi144 ceph-mon[67032]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T23:13:43.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 276886 10.00 MB/sec execute 544 sec latency 1042.256 ms 2024-04-12T23:13:43.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:43 smithi181 ceph-mon[52974]: osdmap e138: 8 total, 8 up, 8 in 2024-04-12T23:13:44.440 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:44 smithi144 ceph-mon[67032]: pgmap v257: 97 pgs: 1 remapped+peering, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 85 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.0 MiB/s wr, 412 op/s; 71/1581 objects degraded (4.491%); 1.3 MiB/s, 27 objects/s recovering 2024-04-12T23:13:44.441 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:44 smithi144 ceph-mon[67032]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T23:13:44.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 277184 9.99 MB/sec execute 545 sec latency 63.129 ms 2024-04-12T23:13:44.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:44 smithi181 ceph-mon[52974]: pgmap v257: 97 pgs: 1 remapped+peering, 6 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 85 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.0 MiB/s wr, 412 op/s; 71/1581 objects degraded (4.491%); 1.3 MiB/s, 27 objects/s recovering 2024-04-12T23:13:44.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:44 smithi181 ceph-mon[52974]: osdmap e139: 8 total, 8 up, 8 in 2024-04-12T23:13:45.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 277553 9.99 MB/sec execute 546 sec latency 57.787 ms 2024-04-12T23:13:46.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 277894 9.99 MB/sec execute 547 sec latency 55.096 ms 2024-04-12T23:13:46.529 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:46 smithi144 ceph-mon[67032]: pgmap v259: 97 pgs: 2 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 90 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.1 MiB/s wr, 340 op/s; 71/1572 objects degraded (4.517%); 950 KiB/s, 28 objects/s recovering 2024-04-12T23:13:46.529 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:46 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 71/1572 objects degraded (4.517%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:46.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:46 smithi181 ceph-mon[52974]: pgmap v259: 97 pgs: 2 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 90 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 3.1 MiB/s wr, 340 op/s; 71/1572 objects degraded (4.517%); 950 KiB/s, 28 objects/s recovering 2024-04-12T23:13:46.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:46 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 71/1572 objects degraded (4.517%), 4 pgs degraded (PG_DEGRADED) 2024-04-12T23:13:47.346 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:13:47.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 278162 9.99 MB/sec execute 548 sec latency 85.234 ms 2024-04-12T23:13:48.370 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:13:48.370 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (8m) 95s ago 17m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (4m) 95s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (4m) 18s ago 16m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 95s ago 17m - - 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (11m) 95s ago 11m 38.5M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (11m) 18s ago 11m 15.2M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (7m) 95s ago 18m 509M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (6m) 18s ago 16m 452M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (6m) 95s ago 18m 51.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (5m) 18s ago 16m 40.6M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (11m) 95s ago 11m 273M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (11m) 18s ago 11m 88.8M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 95s ago 17m 9000k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (7m) 18s ago 16m 17.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (4m) 95s ago 16m 337M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:13:48.371 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (3m) 95s ago 15m 291M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (2m) 95s ago 15m 228M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (103s) 95s ago 14m 12.7M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (60s) 18s ago 14m 260M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (23s) 18s ago 13m 12.1M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (13m) 18s ago 13m 586M 2704M 17.2.0 e1d6a67b021e c16a428ebcb4 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (13m) 18s ago 13m 603M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:13:48.372 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (6m) 95s ago 16m 62.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:13:48.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 278425 9.98 MB/sec execute 549 sec latency 45.826 ms 2024-04-12T23:13:48.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:48 smithi144 ceph-mon[67032]: pgmap v260: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 93 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 54/1635 objects degraded (3.303%); 983 KiB/s, 26 objects/s recovering 2024-04-12T23:13:48.569 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:48 smithi144 ceph-mon[67032]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:48 smithi181 ceph-mon[52974]: pgmap v260: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 93 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.9 MiB/s wr, 422 op/s; 54/1635 objects degraded (3.303%); 983 KiB/s, 26 objects/s recovering 2024-04-12T23:13:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:48 smithi181 ceph-mon[52974]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:13:49.060 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 6 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 10 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:13:49.061 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:13:49.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 278811 9.98 MB/sec execute 550 sec latency 48.046 ms 2024-04-12T23:13:49.597 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:49 smithi144 ceph-mon[67032]: from='client.44339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:49.598 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:49 smithi144 ceph-mon[67032]: from='client.44341 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:49.598 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:49 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/886244887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:49.665 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:13:49.665 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:13:49.665 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:13:49.665 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:13:49.665 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:13:49.665 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "12/23 daemons upgraded", 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:13:49.666 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:13:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:49 smithi181 ceph-mon[52974]: from='client.44339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:49 smithi181 ceph-mon[52974]: from='client.44341 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:49 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/886244887' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:13:50.299 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:13:50.299 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:13:50.299 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:13:50.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 279145 9.98 MB/sec execute 551 sec latency 54.295 ms 2024-04-12T23:13:50.611 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:50 smithi144 ceph-mon[67032]: pgmap v261: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 964 KiB/s, 29 objects/s recovering 2024-04-12T23:13:50.612 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:50 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/1635 objects degraded (3.303%), 3 pgs degraded) 2024-04-12T23:13:50.612 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:50 smithi144 ceph-mon[67032]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:50.612 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:50 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1454315681' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:13:50.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:50 smithi181 ceph-mon[52974]: pgmap v261: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 964 KiB/s, 29 objects/s recovering 2024-04-12T23:13:50.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:50 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/1635 objects degraded (3.303%), 3 pgs degraded) 2024-04-12T23:13:50.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:50 smithi181 ceph-mon[52974]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:13:50.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:50 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1454315681' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:13:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:51 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:51.403 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:51 smithi181 ceph-mon[52974]: Upgrade: osd.6 is safe to restart 2024-04-12T23:13:51.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 279466 9.97 MB/sec execute 552 sec latency 49.385 ms 2024-04-12T23:13:51.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:51.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:51 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-12T23:13:51.516 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:51 smithi144 ceph-mon[67032]: Upgrade: osd.6 is safe to restart 2024-04-12T23:13:52.380 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:52 smithi144 ceph-mon[67032]: pgmap v262: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.9 MiB/s wr, 377 op/s; 354 KiB/s, 15 objects/s recovering 2024-04-12T23:13:52.381 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:52.381 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T23:13:52.381 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:13:52.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 279841 9.97 MB/sec execute 553 sec latency 44.722 ms 2024-04-12T23:13:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:52 smithi181 ceph-mon[52974]: pgmap v262: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.9 MiB/s wr, 377 op/s; 354 KiB/s, 15 objects/s recovering 2024-04-12T23:13:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-12T23:13:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:13:53.414 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:53 smithi144 ceph-mon[67032]: Upgrade: Updating osd.6 2024-04-12T23:13:53.414 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:53 smithi144 ceph-mon[67032]: Deploying daemon osd.6 on smithi181 2024-04-12T23:13:53.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 280295 9.98 MB/sec execute 554 sec latency 41.652 ms 2024-04-12T23:13:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:53 smithi181 ceph-mon[52974]: Upgrade: Updating osd.6 2024-04-12T23:13:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:53 smithi181 ceph-mon[52974]: Deploying daemon osd.6 on smithi181 2024-04-12T23:13:54.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 280642 9.98 MB/sec execute 555 sec latency 79.007 ms 2024-04-12T23:13:54.446 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:54 smithi144 ceph-mon[67032]: pgmap v263: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 316 KiB/s, 14 objects/s recovering 2024-04-12T23:13:54.469 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:54 smithi181 ceph-mon[52974]: pgmap v263: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 474 op/s; 316 KiB/s, 14 objects/s recovering 2024-04-12T23:13:55.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 281022 9.98 MB/sec execute 556 sec latency 37.370 ms 2024-04-12T23:13:56.392 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:56 smithi181 systemd[1]: Stopping Ceph osd.6 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:13:56.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 281332 9.97 MB/sec execute 557 sec latency 57.516 ms 2024-04-12T23:13:56.450 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:56 smithi144 ceph-mon[67032]: pgmap v264: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 476 op/s; 265 KiB/s, 11 objects/s recovering 2024-04-12T23:13:56.450 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:56.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:56.451 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-mon[52974]: pgmap v264: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 476 op/s; 265 KiB/s, 11 objects/s recovering 2024-04-12T23:13:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:13:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:13:56.697 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[37095]: 2024-04-12T23:13:56.426+0000 7f4d768bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:13:56.698 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[37095]: 2024-04-12T23:13:56.426+0000 7f4d768bd700 -1 osd.6 139 *** Got signal Terminated *** 2024-04-12T23:13:56.698 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[37095]: 2024-04-12T23:13:56.426+0000 7f4d768bd700 -1 osd.6 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:13:57.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 281652 9.98 MB/sec execute 558 sec latency 47.041 ms 2024-04-12T23:13:57.484 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:57 smithi144 ceph-mon[67032]: osd.6 marked itself down and dead 2024-04-12T23:13:57.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:57 smithi181 ceph-mon[52974]: osd.6 marked itself down and dead 2024-04-12T23:13:58.350 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:58 smithi181 podman[61975]: 2024-04-12 23:13:58.083161983 +0000 UTC m=+1.783941085 container died c16a428ebcb4fc2e4f08de6e4e016a9037fe1821fe34c3ccc96663e6db89d7fe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, ceph=True, GIT_CLEAN=True, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, version=8, vcs-type=git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870) 2024-04-12T23:13:58.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 281804 9.97 MB/sec execute 559 sec latency 800.362 ms 2024-04-12T23:13:58.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:58 smithi144 ceph-mon[67032]: pgmap v265: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 494 op/s; 245 KiB/s, 7 objects/s recovering 2024-04-12T23:13:58.645 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:58 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:13:58.646 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:58 smithi144 ceph-mon[67032]: osdmap e140: 8 total, 7 up, 8 in 2024-04-12T23:13:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:58 smithi181 ceph-mon[52974]: pgmap v265: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 494 op/s; 245 KiB/s, 7 objects/s recovering 2024-04-12T23:13:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:58 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:13:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:58 smithi181 ceph-mon[52974]: osdmap e140: 8 total, 7 up, 8 in 2024-04-12T23:13:58.698 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:58 smithi181 podman[61975]: 2024-04-12 23:13:58.462313931 +0000 UTC m=+2.163093040 container cleanup c16a428ebcb4fc2e4f08de6e4e016a9037fe1821fe34c3ccc96663e6db89d7fe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, maintainer=Guillaume Abrioux , version=8, architecture=x86_64, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-12T23:13:58.698 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:58 smithi181 bash[61975]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6 2024-04-12T23:13:59.054 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:58 smithi181 podman[62024]: 2024-04-12 23:13:58.714051105 +0000 UTC m=+0.627063222 container remove c16a428ebcb4fc2e4f08de6e4e016a9037fe1821fe34c3ccc96663e6db89d7fe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, CEPH_POINT_RELEASE=-17.2.0, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-type=git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-04-12T23:13:59.305 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:59 smithi181 podman[62112]: 2024-04-12 23:13:58.970077301 +0000 UTC m=+0.020528089 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:13:59.306 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:59 smithi181 podman[62112]: 2024-04-12 23:13:59.081721637 +0000 UTC m=+0.132172394 container create d79595180a546ac7efa998864258fed6b3d7e31bce7740cc21fdf0e9604deaee (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:13:59.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 282143 9.96 MB/sec execute 560 sec latency 764.799 ms 2024-04-12T23:13:59.634 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:13:59 smithi181 ceph-mon[52974]: osdmap e141: 8 total, 7 up, 8 in 2024-04-12T23:13:59.634 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:59 smithi181 podman[62112]: 2024-04-12 23:13:59.31850801 +0000 UTC m=+0.368958781 container init d79595180a546ac7efa998864258fed6b3d7e31bce7740cc21fdf0e9604deaee (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_CLEAN=True) 2024-04-12T23:13:59.635 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:59 smithi181 podman[62112]: 2024-04-12 23:13:59.322305395 +0000 UTC m=+0.372756151 container start d79595180a546ac7efa998864258fed6b3d7e31bce7740cc21fdf0e9604deaee (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T23:13:59.635 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:59 smithi181 podman[62112]: 2024-04-12 23:13:59.41812024 +0000 UTC m=+0.468570997 container attach d79595180a546ac7efa998864258fed6b3d7e31bce7740cc21fdf0e9604deaee (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5) 2024-04-12T23:13:59.661 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:13:59 smithi144 ceph-mon[67032]: osdmap e141: 8 total, 7 up, 8 in 2024-04-12T23:13:59.947 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:13:59 smithi181 podman[62112]: 2024-04-12 23:13:59.647775383 +0000 UTC m=+0.698226149 container died d79595180a546ac7efa998864258fed6b3d7e31bce7740cc21fdf0e9604deaee (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-b277feb, GIT_CLEAN=True) 2024-04-12T23:14:00.421 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:00 smithi144 ceph-mon[67032]: pgmap v268: 97 pgs: 1 active+undersized, 2 active+undersized+degraded, 11 peering, 6 stale+active+clean, 77 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 587 op/s; 23/1641 objects degraded (1.402%) 2024-04-12T23:14:00.421 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:00 smithi144 ceph-mon[67032]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-12T23:14:00.421 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:00 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 23/1641 objects degraded (1.402%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:00.441 INFO:teuthology.orchestra.run.smithi144.stdout: 5 282452 9.96 MB/sec execute 561 sec latency 57.308 ms 2024-04-12T23:14:00.657 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:00 smithi181 ceph-mon[52974]: pgmap v268: 97 pgs: 1 active+undersized, 2 active+undersized+degraded, 11 peering, 6 stale+active+clean, 77 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 587 op/s; 23/1641 objects degraded (1.402%) 2024-04-12T23:14:00.657 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:00 smithi181 ceph-mon[52974]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-04-12T23:14:00.657 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:00 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 23/1641 objects degraded (1.402%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:00.658 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 podman[62138]: 2024-04-12 23:14:00.423065428 +0000 UTC m=+0.768745315 container remove d79595180a546ac7efa998864258fed6b3d7e31bce7740cc21fdf0e9604deaee (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, ceph=True, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-04-12T23:14:00.658 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-12T23:14:00.658 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 systemd[1]: Stopped Ceph osd.6 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:14:00.658 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.6.service: Consumed 5min 19.768s CPU time. 2024-04-12T23:14:00.658 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 systemd[1]: Starting Ceph osd.6 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:14:01.173 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 podman[62242]: 2024-04-12 23:14:00.795631244 +0000 UTC m=+0.020931375 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:14:01.173 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:00 smithi181 podman[62242]: 2024-04-12 23:14:00.924217709 +0000 UTC m=+0.149517830 container create 5a7fbf26e897641d820b189c91c90b3a1976234bd3dd7fbedb408105771db700 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, ceph=True, RELEASE=squid-b277feb) 2024-04-12T23:14:01.173 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 podman[62242]: 2024-04-12 23:14:01.185809763 +0000 UTC m=+0.411109884 container init 5a7fbf26e897641d820b189c91c90b3a1976234bd3dd7fbedb408105771db700 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T23:14:01.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 282706 9.96 MB/sec execute 562 sec latency 57.244 ms 2024-04-12T23:14:01.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 podman[62242]: 2024-04-12 23:14:01.189824417 +0000 UTC m=+0.415124542 container start 5a7fbf26e897641d820b189c91c90b3a1976234bd3dd7fbedb408105771db700 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=) 2024-04-12T23:14:01.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 podman[62242]: 2024-04-12 23:14:01.285927306 +0000 UTC m=+0.511227449 container attach 5a7fbf26e897641d820b189c91c90b3a1976234bd3dd7fbedb408105771db700 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-12T23:14:01.947 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 bash[62242]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-12T23:14:01.948 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-12T23:14:01.949 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate[62257]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-12T23:14:01.949 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:01 smithi181 podman[62242]: 2024-04-12 23:14:01.714557537 +0000 UTC m=+0.939857673 container died 5a7fbf26e897641d820b189c91c90b3a1976234bd3dd7fbedb408105771db700 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409) 2024-04-12T23:14:02.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 282994 9.95 MB/sec execute 563 sec latency 74.068 ms 2024-04-12T23:14:02.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:02 smithi181 ceph-mon[52974]: pgmap v269: 97 pgs: 3 active+undersized, 8 active+undersized+degraded, 11 peering, 4 stale+active+clean, 71 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.7 MiB/s wr, 451 op/s; 75/1647 objects degraded (4.554%) 2024-04-12T23:14:02.460 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:02 smithi144 ceph-mon[67032]: pgmap v269: 97 pgs: 3 active+undersized, 8 active+undersized+degraded, 11 peering, 4 stale+active+clean, 71 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.7 MiB/s wr, 451 op/s; 75/1647 objects degraded (4.554%) 2024-04-12T23:14:02.846 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:02 smithi181 podman[62374]: 2024-04-12 23:14:02.600096758 +0000 UTC m=+0.879520944 container remove 5a7fbf26e897641d820b189c91c90b3a1976234bd3dd7fbedb408105771db700 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-12T23:14:02.846 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:02 smithi181 podman[62437]: 2024-04-12 23:14:02.732943713 +0000 UTC m=+0.020217772 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:14:02.846 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:02 smithi181 podman[62437]: 2024-04-12 23:14:02.859389567 +0000 UTC m=+0.146663614 container create 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T23:14:03.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 283436 9.95 MB/sec execute 564 sec latency 38.905 ms 2024-04-12T23:14:03.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:03 smithi181 podman[62437]: 2024-04-12 23:14:03.112298731 +0000 UTC m=+0.399572786 container init 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-12T23:14:03.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:03 smithi181 podman[62437]: 2024-04-12 23:14:03.116452731 +0000 UTC m=+0.403726780 container start 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:14:03.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:03 smithi181 ceph-osd[62456]: -- 172.21.15.181:0/3408268298 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562a5502d680 con 0x562a54ff6c00 2024-04-12T23:14:03.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:03 smithi181 bash[62437]: 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 2024-04-12T23:14:03.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:03 smithi181 systemd[1]: Started Ceph osd.6 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:14:03.947 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:14:03.657+0000 7f09ad8277c0 -1 Falling back to public interface 2024-04-12T23:14:04.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 283847 9.95 MB/sec execute 565 sec latency 36.326 ms 2024-04-12T23:14:04.482 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:04.483 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:04.483 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:04.483 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:04 smithi144 ceph-mon[67032]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 201/1650 objects degraded (12.182%) 2024-04-12T23:14:04.495 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:04.495 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:04.495 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:04.495 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:04 smithi181 ceph-mon[52974]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 201/1650 objects degraded (12.182%) 2024-04-12T23:14:05.246 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:05 smithi144 ceph-mon[67032]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T23:14:05.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 284298 9.96 MB/sec execute 566 sec latency 44.553 ms 2024-04-12T23:14:05.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:05 smithi181 ceph-mon[52974]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-04-12T23:14:06.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:06 smithi144 ceph-mon[67032]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 192/1620 objects degraded (11.852%) 2024-04-12T23:14:06.268 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:06 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 201/1650 objects degraded (12.182%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:06.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 284729 9.96 MB/sec execute 567 sec latency 49.215 ms 2024-04-12T23:14:06.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:06 smithi181 ceph-mon[52974]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.0 MiB/s wr, 516 op/s; 192/1620 objects degraded (11.852%) 2024-04-12T23:14:06.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:06 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 201/1650 objects degraded (12.182%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:07.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 285168 9.97 MB/sec execute 568 sec latency 38.566 ms 2024-04-12T23:14:08.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 285475 9.96 MB/sec execute 569 sec latency 67.123 ms 2024-04-12T23:14:08.542 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:08 smithi144 ceph-mon[67032]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 195/1632 objects degraded (11.949%) 2024-04-12T23:14:08.543 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:08.543 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:08.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:08 smithi181 ceph-mon[52974]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 195/1632 objects degraded (11.949%) 2024-04-12T23:14:08.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:08.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:09.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 285840 9.96 MB/sec execute 570 sec latency 48.755 ms 2024-04-12T23:14:10.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:10.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:10.333 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:10 smithi144 ceph-mon[67032]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 572 op/s; 200/1677 objects degraded (11.926%) 2024-04-12T23:14:10.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 286114 9.96 MB/sec execute 571 sec latency 50.285 ms 2024-04-12T23:14:10.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:10.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:10.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:10 smithi181 ceph-mon[52974]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 572 op/s; 200/1677 objects degraded (11.926%) 2024-04-12T23:14:11.365 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:11 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 200/1677 objects degraded (11.926%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:11.366 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:11.366 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:11.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 286472 9.96 MB/sec execute 572 sec latency 64.831 ms 2024-04-12T23:14:11.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:11 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 200/1677 objects degraded (11.926%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:11.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:11.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:11.448 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:11 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:14:11.337+0000 7f09ad8277c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-12T23:14:12.384 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:12 smithi144 ceph-mon[67032]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.4 MiB/s wr, 537 op/s; 201/1689 objects degraded (11.901%) 2024-04-12T23:14:12.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 286873 9.96 MB/sec execute 573 sec latency 31.548 ms 2024-04-12T23:14:12.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:12 smithi181 ceph-mon[52974]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.4 MiB/s wr, 537 op/s; 201/1689 objects degraded (11.901%) 2024-04-12T23:14:12.947 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:12 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:14:12.848+0000 7f09ad8277c0 -1 osd.6 139 log_to_monitors true 2024-04-12T23:14:13.411 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:13 smithi144 ceph-mon[67032]: from='osd.6 [v2:172.21.15.181:6816/318193237,v1:172.21.15.181:6817/318193237]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:14:13.411 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:13 smithi144 ceph-mon[67032]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:14:13.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 287195 9.95 MB/sec execute 574 sec latency 90.490 ms 2024-04-12T23:14:13.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:13 smithi181 ceph-mon[52974]: from='osd.6 [v2:172.21.15.181:6816/318193237,v1:172.21.15.181:6817/318193237]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:14:13.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:13 smithi181 ceph-mon[52974]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-12T23:14:13.780 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:14:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:14:13.689+0000 7f099ac77700 -1 osd.6 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: osdmap e142: 8 total, 7 up, 8 in 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='osd.6 [v2:172.21.15.181:6816/318193237,v1:172.21.15.181:6817/318193237]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 507 op/s; 196/1644 objects degraded (11.922%) 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.426 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.8 MiB/s wr, 569 op/s; 199/1650 objects degraded (12.061%) 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='osd.6 ' entity='osd.6' 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:14.427 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:14.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 287523 9.95 MB/sec execute 575 sec latency 75.007 ms 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: osdmap e142: 8 total, 7 up, 8 in 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='osd.6 [v2:172.21.15.181:6816/318193237,v1:172.21.15.181:6817/318193237]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 507 op/s; 196/1644 objects degraded (11.922%) 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.8 MiB/s wr, 569 op/s; 199/1650 objects degraded (12.061%) 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='osd.6 ' entity='osd.6' 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:14.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:15.205 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:15 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:15.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:15 smithi144 ceph-mon[67032]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-12T23:14:15.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:15 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:14:15.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:15 smithi144 ceph-mon[67032]: osd.6 [v2:172.21.15.181:6816/318193237,v1:172.21.15.181:6817/318193237] boot 2024-04-12T23:14:15.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:15 smithi144 ceph-mon[67032]: osdmap e143: 8 total, 8 up, 8 in 2024-04-12T23:14:15.206 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:14:15.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 287775 9.94 MB/sec execute 576 sec latency 738.906 ms 2024-04-12T23:14:15.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:15 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:15.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:15 smithi181 ceph-mon[52974]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-04-12T23:14:15.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:15 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:14:15.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:15 smithi181 ceph-mon[52974]: osd.6 [v2:172.21.15.181:6816/318193237,v1:172.21.15.181:6817/318193237] boot 2024-04-12T23:14:15.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:15 smithi181 ceph-mon[52974]: osdmap e143: 8 total, 8 up, 8 in 2024-04-12T23:14:15.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-12T23:14:16.262 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:16 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 199/1650 objects degraded (12.061%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:16.263 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:16 smithi144 ceph-mon[67032]: osdmap e144: 8 total, 8 up, 8 in 2024-04-12T23:14:16.263 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:16 smithi144 ceph-mon[67032]: pgmap v280: 97 pgs: 1 active+recovery_wait+degraded, 3 remapped+peering, 8 active+undersized, 20 active+undersized+degraded, 65 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 573 op/s; 182/1656 objects degraded (10.990%); 948 B/s, 0 objects/s recovering 2024-04-12T23:14:16.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 288030 9.94 MB/sec execute 577 sec latency 1049.408 ms 2024-04-12T23:14:16.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:16 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 199/1650 objects degraded (12.061%), 25 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:16.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:16 smithi181 ceph-mon[52974]: osdmap e144: 8 total, 8 up, 8 in 2024-04-12T23:14:16.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:16 smithi181 ceph-mon[52974]: pgmap v280: 97 pgs: 1 active+recovery_wait+degraded, 3 remapped+peering, 8 active+undersized, 20 active+undersized+degraded, 65 active+clean; 348 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 573 op/s; 182/1656 objects degraded (10.990%); 948 B/s, 0 objects/s recovering 2024-04-12T23:14:17.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 288350 9.94 MB/sec execute 578 sec latency 1101.243 ms 2024-04-12T23:14:17.557 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:17 smithi144 ceph-mon[67032]: osdmap e145: 8 total, 8 up, 8 in 2024-04-12T23:14:17.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:17 smithi181 ceph-mon[52974]: osdmap e145: 8 total, 8 up, 8 in 2024-04-12T23:14:18.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 288676 9.94 MB/sec execute 579 sec latency 734.612 ms 2024-04-12T23:14:18.570 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:18 smithi144 ceph-mon[67032]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T23:14:18.571 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:18 smithi144 ceph-mon[67032]: pgmap v283: 97 pgs: 1 active+recovery_wait+degraded, 3 remapped+peering, 7 active+undersized, 4 active+undersized+remapped, 11 active+undersized+degraded, 71 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.0 MiB/s wr, 562 op/s; 107/1632 objects degraded (6.556%); 2.2 MiB/s, 26 objects/s recovering 2024-04-12T23:14:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:18 smithi181 ceph-mon[52974]: osdmap e146: 8 total, 8 up, 8 in 2024-04-12T23:14:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:18 smithi181 ceph-mon[52974]: pgmap v283: 97 pgs: 1 active+recovery_wait+degraded, 3 remapped+peering, 7 active+undersized, 4 active+undersized+remapped, 11 active+undersized+degraded, 71 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.0 MiB/s wr, 562 op/s; 107/1632 objects degraded (6.556%); 2.2 MiB/s, 26 objects/s recovering 2024-04-12T23:14:19.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 289328 9.95 MB/sec execute 580 sec latency 761.771 ms 2024-04-12T23:14:19.586 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:19 smithi144 ceph-mon[67032]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T23:14:19.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:19 smithi181 ceph-mon[52974]: osdmap e147: 8 total, 8 up, 8 in 2024-04-12T23:14:20.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 289883 9.96 MB/sec execute 581 sec latency 44.234 ms 2024-04-12T23:14:20.641 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:20 smithi144 ceph-mon[67032]: pgmap v285: 97 pgs: 2 peering, 1 active+recovery_wait+degraded, 2 remapped+peering, 3 active+undersized+remapped, 89 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 568 op/s; 22/1614 objects degraded (1.363%); 585 KiB/s, 64 keys/s, 46 objects/s recovering 2024-04-12T23:14:20.641 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:20 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 107/1632 objects degraded (6.556%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:20.805 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:14:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:20 smithi181 ceph-mon[52974]: pgmap v285: 97 pgs: 2 peering, 1 active+recovery_wait+degraded, 2 remapped+peering, 3 active+undersized+remapped, 89 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 568 op/s; 22/1614 objects degraded (1.363%); 585 KiB/s, 64 keys/s, 46 objects/s recovering 2024-04-12T23:14:20.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:20 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 107/1632 objects degraded (6.556%), 12 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:21.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 290250 9.96 MB/sec execute 582 sec latency 59.687 ms 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (9m) 2m ago 18m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (5m) 2m ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (5m) 14s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 2m ago 17m - - 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (12m) 2m ago 12m 38.5M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (12m) 14s ago 12m 15.3M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (8m) 2m ago 19m 509M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (7m) 14s ago 17m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (6m) 2m ago 19m 51.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:14:21.876 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (6m) 14s ago 17m 42.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (11m) 2m ago 11m 273M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (11m) 14s ago 11m 88.9M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (8m) 2m ago 17m 9000k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 14s ago 17m 17.0M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (4m) 2m ago 16m 337M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (3m) 2m ago 16m 291M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (3m) 2m ago 15m 228M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (2m) 2m ago 15m 12.7M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (94s) 14s ago 14m 305M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (57s) 14s ago 14m 258M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (19s) 14s ago 14m 15.1M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (13m) 14s ago 13m 649M 2704M 17.2.0 e1d6a67b021e fc896fd9599d 2024-04-12T23:14:21.877 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (7m) 2m ago 16m 62.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:14:21.910 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:21 smithi144 ceph-mon[67032]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:21.910 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:21 smithi144 ceph-mon[67032]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:21.910 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:21 smithi144 ceph-mon[67032]: pgmap v286: 97 pgs: 2 peering, 3 active+undersized+remapped, 92 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 285 op/s; 1.3 MiB/s, 44 keys/s, 38 objects/s recovering 2024-04-12T23:14:21.911 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:21 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1614 objects degraded (1.363%), 1 pg degraded) 2024-04-12T23:14:22.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:21 smithi181 ceph-mon[52974]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:22.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:21 smithi181 ceph-mon[52974]: from='client.44363 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:22.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:21 smithi181 ceph-mon[52974]: pgmap v286: 97 pgs: 2 peering, 3 active+undersized+remapped, 92 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 285 op/s; 1.3 MiB/s, 44 keys/s, 38 objects/s recovering 2024-04-12T23:14:22.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:21 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1614 objects degraded (1.363%), 1 pg degraded) 2024-04-12T23:14:22.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 290587 9.96 MB/sec execute 583 sec latency 53.805 ms 2024-04-12T23:14:22.495 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:14:22.495 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 7 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:22.496 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:14:22.497 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-12T23:14:22.497 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 11 2024-04-12T23:14:22.497 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:14:22.497 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:14:22.939 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:22 smithi144 ceph-mon[67032]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:22.940 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:22 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2813393053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:22.973 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:14:22.973 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:14:22.973 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:14:22.973 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "13/23 daemons upgraded", 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading osd daemons", 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:14:22.974 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:14:23.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:22 smithi181 ceph-mon[52974]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:23.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:22 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2813393053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:23.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 291004 9.96 MB/sec execute 584 sec latency 51.018 ms 2024-04-12T23:14:23.676 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:14:23.676 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:14:23.676 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:14:23.952 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:23 smithi144 ceph-mon[67032]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:23.952 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:23 smithi144 ceph-mon[67032]: pgmap v287: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 630 op/s; 2.7 MiB/s, 37 keys/s, 32 objects/s recovering 2024-04-12T23:14:23.952 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:23 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1918791730' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:14:24.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:23 smithi181 ceph-mon[52974]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:24.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:23 smithi181 ceph-mon[52974]: pgmap v287: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 630 op/s; 2.7 MiB/s, 37 keys/s, 32 objects/s recovering 2024-04-12T23:14:24.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:23 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1918791730' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:14:24.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 291364 9.96 MB/sec execute 585 sec latency 38.383 ms 2024-04-12T23:14:25.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 291764 9.96 MB/sec execute 586 sec latency 57.759 ms 2024-04-12T23:14:26.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 292121 9.96 MB/sec execute 587 sec latency 50.810 ms 2024-04-12T23:14:26.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:26 smithi144 ceph-mon[67032]: pgmap v288: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.7 MiB/s wr, 559 op/s; 1.3 MiB/s, 32 keys/s, 16 objects/s recovering 2024-04-12T23:14:26.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:26.921 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:26 smithi181 ceph-mon[52974]: pgmap v288: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.7 MiB/s wr, 559 op/s; 1.3 MiB/s, 32 keys/s, 16 objects/s recovering 2024-04-12T23:14:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:26.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:27.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 292527 9.97 MB/sec execute 588 sec latency 64.446 ms 2024-04-12T23:14:28.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 292794 9.96 MB/sec execute 589 sec latency 68.234 ms 2024-04-12T23:14:28.713 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:28 smithi144 ceph-mon[67032]: pgmap v289: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 496 op/s; 1.1 MiB/s, 26 keys/s, 13 objects/s recovering 2024-04-12T23:14:28.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:28 smithi181 ceph-mon[52974]: pgmap v289: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 496 op/s; 1.1 MiB/s, 26 keys/s, 13 objects/s recovering 2024-04-12T23:14:29.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 293116 9.96 MB/sec execute 590 sec latency 43.386 ms 2024-04-12T23:14:29.727 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:29 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:29.777 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:29 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:30.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 293497 9.96 MB/sec execute 591 sec latency 62.784 ms 2024-04-12T23:14:30.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:30 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:30.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:30 smithi144 ceph-mon[67032]: Upgrade: osd.7 is safe to restart 2024-04-12T23:14:30.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:30 smithi144 ceph-mon[67032]: pgmap v290: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.6 MiB/s wr, 607 op/s; 1.5 MiB/s, 3 objects/s recovering 2024-04-12T23:14:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:30 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-12T23:14:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:30 smithi181 ceph-mon[52974]: Upgrade: osd.7 is safe to restart 2024-04-12T23:14:30.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:30 smithi181 ceph-mon[52974]: pgmap v290: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.6 MiB/s wr, 607 op/s; 1.5 MiB/s, 3 objects/s recovering 2024-04-12T23:14:31.442 INFO:teuthology.orchestra.run.smithi144.stdout: 5 293856 9.96 MB/sec execute 592 sec latency 44.628 ms 2024-04-12T23:14:32.030 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: Upgrade: Updating osd.7 2024-04-12T23:14:32.031 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:32.031 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T23:14:32.031 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:32.031 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: Deploying daemon osd.7 on smithi181 2024-04-12T23:14:32.031 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:32.031 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:31 smithi144 ceph-mon[67032]: pgmap v291: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.8 MiB/s wr, 591 op/s; 1.4 MiB/s, 3 objects/s recovering 2024-04-12T23:14:32.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: Upgrade: Updating osd.7 2024-04-12T23:14:32.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:32.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-12T23:14:32.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:32.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: Deploying daemon osd.7 on smithi181 2024-04-12T23:14:32.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:32.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:31 smithi181 ceph-mon[52974]: pgmap v291: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.8 MiB/s wr, 591 op/s; 1.4 MiB/s, 3 objects/s recovering 2024-04-12T23:14:32.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 294204 9.96 MB/sec execute 593 sec latency 52.533 ms 2024-04-12T23:14:33.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 294600 9.96 MB/sec execute 594 sec latency 51.026 ms 2024-04-12T23:14:34.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 294925 9.95 MB/sec execute 595 sec latency 33.453 ms 2024-04-12T23:14:34.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:34 smithi181 ceph-mon[52974]: pgmap v292: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 662 op/s; 1024 KiB/s, 0 objects/s recovering 2024-04-12T23:14:34.842 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:34 smithi144 ceph-mon[67032]: pgmap v292: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 662 op/s; 1024 KiB/s, 0 objects/s recovering 2024-04-12T23:14:35.173 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:34 smithi181 systemd[1]: Stopping Ceph osd.7 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:14:35.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 295240 9.95 MB/sec execute 596 sec latency 50.301 ms 2024-04-12T23:14:35.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:35 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[40176]: 2024-04-12T23:14:35.191+0000 7fde5987a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:14:35.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:35 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[40176]: 2024-04-12T23:14:35.191+0000 7fde5987a700 -1 osd.7 147 *** Got signal Terminated *** 2024-04-12T23:14:35.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:35 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[40176]: 2024-04-12T23:14:35.191+0000 7fde5987a700 -1 osd.7 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:14:35.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:35 smithi181 ceph-mon[52974]: osd.7 marked itself down and dead 2024-04-12T23:14:36.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:35 smithi144 ceph-mon[67032]: osd.7 marked itself down and dead 2024-04-12T23:14:36.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 295317 9.94 MB/sec execute 597 sec latency 775.899 ms 2024-04-12T23:14:36.680 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:36 smithi181 podman[64228]: 2024-04-12 23:14:36.396616496 +0000 UTC m=+1.338446288 container died fc896fd9599d11d108aac943971abd06d9b1415850d7903682dc8cf0766e8879 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, maintainer=Guillaume Abrioux , vcs-type=git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, name=centos-stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-12T23:14:36.929 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:36 smithi144 ceph-mon[67032]: pgmap v293: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-04-12T23:14:36.930 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:36 smithi144 ceph-mon[67032]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:14:36.930 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:36 smithi144 ceph-mon[67032]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-12T23:14:36.930 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:36 smithi144 ceph-mon[67032]: osdmap e148: 8 total, 7 up, 8 in 2024-04-12T23:14:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:36 smithi181 ceph-mon[52974]: pgmap v293: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-04-12T23:14:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:36 smithi181 ceph-mon[52974]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-12T23:14:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:36 smithi181 ceph-mon[52974]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-12T23:14:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:36 smithi181 ceph-mon[52974]: osdmap e148: 8 total, 7 up, 8 in 2024-04-12T23:14:36.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:36 smithi181 podman[64228]: 2024-04-12 23:14:36.791081072 +0000 UTC m=+1.732910875 container cleanup fc896fd9599d11d108aac943971abd06d9b1415850d7903682dc8cf0766e8879 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, RELEASE=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, release=754, vcs-type=git, version=8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, ceph=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-12T23:14:36.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:36 smithi181 bash[64228]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7 2024-04-12T23:14:37.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 295648 9.94 MB/sec execute 598 sec latency 1032.252 ms 2024-04-12T23:14:37.576 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:37 smithi181 podman[64255]: 2024-04-12 23:14:37.251272859 +0000 UTC m=+0.855119765 container remove fc896fd9599d11d108aac943971abd06d9b1415850d7903682dc8cf0766e8879 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, vcs-type=git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, release=754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, RELEASE=HEAD) 2024-04-12T23:14:37.707 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:37 smithi144 ceph-mon[67032]: osdmap e149: 8 total, 7 up, 8 in 2024-04-12T23:14:37.917 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:37 smithi181 ceph-mon[52974]: osdmap e149: 8 total, 7 up, 8 in 2024-04-12T23:14:37.918 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:37 smithi181 podman[64379]: 2024-04-12 23:14:37.491750425 +0000 UTC m=+0.020673956 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:14:37.918 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:37 smithi181 podman[64379]: 2024-04-12 23:14:37.594065372 +0000 UTC m=+0.122988882 container create 71bdd5ea9d13cb4f85fbc2161cfda8169661fcbb49d641974e737bf553f9b4bb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:14:37.918 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:37 smithi181 podman[64379]: 2024-04-12 23:14:37.831006393 +0000 UTC m=+0.359929920 container init 71bdd5ea9d13cb4f85fbc2161cfda8169661fcbb49d641974e737bf553f9b4bb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T23:14:37.918 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:37 smithi181 podman[64379]: 2024-04-12 23:14:37.835062614 +0000 UTC m=+0.363986133 container start 71bdd5ea9d13cb4f85fbc2161cfda8169661fcbb49d641974e737bf553f9b4bb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240409) 2024-04-12T23:14:37.918 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:37 smithi181 podman[64379]: 2024-04-12 23:14:37.93078383 +0000 UTC m=+0.459707370 container attach 71bdd5ea9d13cb4f85fbc2161cfda8169661fcbb49d641974e737bf553f9b4bb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-12T23:14:38.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 296113 9.95 MB/sec execute 599 sec latency 34.690 ms 2024-04-12T23:14:38.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:38 smithi181 podman[64379]: 2024-04-12 23:14:38.19132758 +0000 UTC m=+0.720251092 container died 71bdd5ea9d13cb4f85fbc2161cfda8169661fcbb49d641974e737bf553f9b4bb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-12T23:14:38.735 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:38 smithi144 ceph-mon[67032]: pgmap v296: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 11 stale+active+clean, 77 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 42/1665 objects degraded (2.523%) 2024-04-12T23:14:38.735 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:38 smithi144 ceph-mon[67032]: Health check failed: Degraded data redundancy: 42/1665 objects degraded (2.523%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:38 smithi181 ceph-mon[52974]: pgmap v296: 97 pgs: 2 active+undersized, 7 active+undersized+degraded, 11 stale+active+clean, 77 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.8 MiB/s wr, 479 op/s; 42/1665 objects degraded (2.523%) 2024-04-12T23:14:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:38 smithi181 ceph-mon[52974]: Health check failed: Degraded data redundancy: 42/1665 objects degraded (2.523%), 7 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:39.443 INFO:teuthology.orchestra.run.smithi144.stdout: 5 cleanup 600 sec 2024-04-12T23:14:39.668 INFO:teuthology.orchestra.run.smithi144.stdout: 0 cleanup 600 sec 2024-04-12T23:14:39.668 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Operation Count AvgLat MaxLat 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: ---------------------------------------- 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: NTCreateX 191769 4.525 2334.088 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Close 140733 0.134 10.996 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Rename 8117 9.686 1075.984 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Unlink 38789 6.788 2301.238 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Qpathinfo 173914 1.200 2576.721 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Qfileinfo 30285 0.008 0.788 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Qfsinfo 31837 0.596 45.125 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Sfileinfo 15634 12.411 1019.691 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: Find 67126 0.943 11.649 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: WriteX 94669 13.513 2583.168 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: ReadX 300108 0.010 8.504 2024-04-12T23:14:39.669 INFO:teuthology.orchestra.run.smithi144.stdout: LockX 622 0.358 3.152 2024-04-12T23:14:39.670 INFO:teuthology.orchestra.run.smithi144.stdout: UnlockX 622 0.315 1.930 2024-04-12T23:14:39.670 INFO:teuthology.orchestra.run.smithi144.stdout: Flush 13437 0.002 0.025 2024-04-12T23:14:39.670 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:14:39.670 INFO:teuthology.orchestra.run.smithi144.stdout:Throughput 9.94683 MB/sec 5 clients 5 procs max_latency=2583.173 ms 2024-04-12T23:14:39.671 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-04-12T23:14:39.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 podman[64405]: 2024-04-12 23:14:39.452853014 +0000 UTC m=+1.251936835 container remove 71bdd5ea9d13cb4f85fbc2161cfda8169661fcbb49d641974e737bf553f9b4bb (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-12T23:14:39.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-12T23:14:39.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 systemd[1]: Stopped Ceph osd.7 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:14:39.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.7.service: Consumed 7min 39.054s CPU time. 2024-04-12T23:14:39.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 systemd[1]: Starting Ceph osd.7 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:14:39.712 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-04-12T23:14:39.846 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-04-12T23:14:39.873 INFO:teuthology.orchestra.run.smithi144.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-04-12T23:14:39.874 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:14:39.874 INFO:teuthology.orchestra.run.smithi144.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-04-12T23:14:39.874 INFO:teuthology.orchestra.run.smithi144.stdout:3 of 5 processes prepared for launch 0 sec 2024-04-12T23:14:39.974 INFO:teuthology.orchestra.run.smithi144.stdout:5 of 5 processes prepared for launch 0 sec 2024-04-12T23:14:39.975 INFO:teuthology.orchestra.run.smithi144.stdout:releasing clients 2024-04-12T23:14:40.186 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 podman[64510]: 2024-04-12 23:14:39.799174166 +0000 UTC m=+0.018059358 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:14:40.186 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:39 smithi181 podman[64510]: 2024-04-12 23:14:39.921376451 +0000 UTC m=+0.140261621 container create 0c13c502cfb08f0769b7cbf192143f3c8d2ad90a7df87ddd50d5f4630c2576f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-12T23:14:40.186 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 podman[64510]: 2024-04-12 23:14:40.124517291 +0000 UTC m=+0.343402462 container init 0c13c502cfb08f0769b7cbf192143f3c8d2ad90a7df87ddd50d5f4630c2576f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=) 2024-04-12T23:14:40.186 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 podman[64510]: 2024-04-12 23:14:40.128405843 +0000 UTC m=+0.347291014 container start 0c13c502cfb08f0769b7cbf192143f3c8d2ad90a7df87ddd50d5f4630c2576f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409) 2024-04-12T23:14:40.186 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 podman[64510]: 2024-04-12 23:14:40.199197495 +0000 UTC m=+0.418082693 container attach 0c13c502cfb08f0769b7cbf192143f3c8d2ad90a7df87ddd50d5f4630c2576f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:14:40.826 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:40 smithi144 ceph-mon[67032]: pgmap v297: 97 pgs: 2 active+undersized, 23 active+undersized+degraded, 4 stale+active+clean, 68 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 556 op/s; 146/1629 objects degraded (8.963%) 2024-04-12T23:14:40.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-mon[52974]: pgmap v297: 97 pgs: 2 active+undersized, 23 active+undersized+degraded, 4 stale+active+clean, 68 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.0 MiB/s wr, 556 op/s; 146/1629 objects degraded (8.963%) 2024-04-12T23:14:40.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T23:14:40.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T23:14:40.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T23:14:40.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-12T23:14:40.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T23:14:40.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 bash[64510]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate[64525]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-12T23:14:40.948 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:40 smithi181 podman[64510]: 2024-04-12 23:14:40.713156041 +0000 UTC m=+0.932041210 container died 0c13c502cfb08f0769b7cbf192143f3c8d2ad90a7df87ddd50d5f4630c2576f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb) 2024-04-12T23:14:41.634 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 podman[64647]: 2024-04-12 23:14:41.354283304 +0000 UTC m=+0.635257538 container remove 0c13c502cfb08f0769b7cbf192143f3c8d2ad90a7df87ddd50d5f4630c2576f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-activate, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b277feb, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:14:41.634 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 podman[64707]: 2024-04-12 23:14:41.488424848 +0000 UTC m=+0.020132522 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:14:41.634 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 podman[64707]: 2024-04-12 23:14:41.647413599 +0000 UTC m=+0.179121253 container create 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, io.buildah.version=1.33.5, RELEASE=squid-b277feb, ceph=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-12T23:14:41.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 podman[64707]: 2024-04-12 23:14:41.868042979 +0000 UTC m=+0.399750640 container init 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux ) 2024-04-12T23:14:41.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 podman[64707]: 2024-04-12 23:14:41.871444835 +0000 UTC m=+0.403152491 container start 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-12T23:14:41.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 ceph-osd[64725]: -- 172.21.15.181:0/1456990030 <== mon.1 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557046f57680 con 0x557046f21000 2024-04-12T23:14:41.975 INFO:teuthology.orchestra.run.smithi144.stdout: 5 446 29.65 MB/sec execute 1 sec latency 52.200 ms 2024-04-12T23:14:42.088 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:42.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:42.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:42 smithi144 ceph-mon[67032]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 356 op/s; 236/1638 objects degraded (14.408%) 2024-04-12T23:14:42.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:42.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:42.089 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:42.401 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 bash[64707]: 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 2024-04-12T23:14:42.401 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:41 smithi181 systemd[1]: Started Ceph osd.7 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:14:42.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:42.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:42.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-mon[52974]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 356 op/s; 236/1638 objects degraded (14.408%) 2024-04-12T23:14:42.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:42.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:42.402 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:42.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:42 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:14:42.413+0000 7fe5b1a697c0 -1 Falling back to public interface 2024-04-12T23:14:42.975 INFO:teuthology.orchestra.run.smithi144.stdout: 5 651 28.33 MB/sec execute 2 sec latency 74.565 ms 2024-04-12T23:14:43.881 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:43 smithi144 ceph-mon[67032]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 12 MiB/s wr, 467 op/s; 217/1479 objects degraded (14.672%) 2024-04-12T23:14:43.882 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:43 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 217/1479 objects degraded (14.672%), 35 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:43.975 INFO:teuthology.orchestra.run.smithi144.stdout: 5 960 20.09 MB/sec execute 3 sec latency 34.301 ms 2024-04-12T23:14:44.095 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:43 smithi181 ceph-mon[52974]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 12 MiB/s wr, 467 op/s; 217/1479 objects degraded (14.672%) 2024-04-12T23:14:44.095 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:43 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 217/1479 objects degraded (14.672%), 35 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:44.980 INFO:teuthology.orchestra.run.smithi144.stdout: 5 1423 20.28 MB/sec execute 4 sec latency 56.961 ms 2024-04-12T23:14:45.981 INFO:teuthology.orchestra.run.smithi144.stdout: 5 cleanup 5 sec 2024-04-12T23:14:46.272 INFO:teuthology.orchestra.run.smithi144.stdout: 0 cleanup 5 sec 2024-04-12T23:14:46.272 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:14:46.272 INFO:teuthology.orchestra.run.smithi144.stdout: Operation Count AvgLat MaxLat 2024-04-12T23:14:46.272 INFO:teuthology.orchestra.run.smithi144.stdout: ---------------------------------------- 2024-04-12T23:14:46.272 INFO:teuthology.orchestra.run.smithi144.stdout: NTCreateX 1409 4.587 55.618 2024-04-12T23:14:46.272 INFO:teuthology.orchestra.run.smithi144.stdout: Close 1165 4.966 74.554 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Rename 55 10.036 34.295 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Unlink 123 4.885 20.083 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Qpathinfo 974 1.429 22.493 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Qfileinfo 430 13.943 66.550 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Qfsinfo 165 0.407 2.871 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Sfileinfo 170 12.706 23.196 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Find 352 0.692 3.576 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: WriteX 1697 0.022 0.074 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: ReadX 1463 0.010 0.235 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: Flush 140 12.775 26.073 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:14:46.273 INFO:teuthology.orchestra.run.smithi144.stdout:Throughput 20.2845 MB/sec 5 clients 5 procs max_latency=74.565 ms 2024-04-12T23:14:46.274 DEBUG:teuthology.parallel:result is None 2024-04-12T23:14:46.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:46 smithi181 ceph-mon[52974]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 13 MiB/s wr, 427 op/s; 201/1440 objects degraded (13.958%) 2024-04-12T23:14:47.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:46 smithi144 ceph-mon[67032]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 13 MiB/s wr, 427 op/s; 201/1440 objects degraded (13.958%) 2024-04-12T23:14:48.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:47 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:48.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:47 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:48.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:47 smithi181 ceph-mon[52974]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 13 MiB/s wr, 516 op/s; 203/1446 objects degraded (14.039%) 2024-04-12T23:14:48.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:47 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:48.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:47 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:48.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:47 smithi144 ceph-mon[67032]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 13 MiB/s wr, 516 op/s; 203/1446 objects degraded (14.039%) 2024-04-12T23:14:49.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:49.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:49.438 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:49.438 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:50.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:49 smithi181 ceph-mon[52974]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 309 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 484 op/s; 190/1251 objects degraded (15.188%) 2024-04-12T23:14:50.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:49 smithi144 ceph-mon[67032]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 309 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 12 MiB/s wr, 484 op/s; 190/1251 objects degraded (15.188%) 2024-04-12T23:14:51.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:50 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 190/1251 objects degraded (15.188%), 35 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:51.197 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:50 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:14:50.987+0000 7fe5b1a697c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-12T23:14:51.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:50 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 190/1251 objects degraded (15.188%), 35 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:14:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: pgmap v303: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 267 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 12 MiB/s wr, 436 op/s; 134/795 objects degraded (16.855%) 2024-04-12T23:14:52.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: pgmap v304: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 267 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 485 op/s; 134/795 objects degraded (16.855%) 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all osd 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-12T23:14:52.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: Upgrade: Setting require_osd_release to 19 squid 2024-04-12T23:14:52.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-12T23:14:52.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: pgmap v303: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 267 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 12 MiB/s wr, 436 op/s; 134/795 objects degraded (16.855%) 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: pgmap v304: 97 pgs: 10 active+undersized, 32 active+undersized+degraded, 55 active+clean; 267 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 12 MiB/s wr, 485 op/s; 134/795 objects degraded (16.855%) 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:52.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all osd 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-12T23:14:52.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-12T23:14:52.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: Upgrade: Setting require_osd_release to 19 squid 2024-04-12T23:14:52.776 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-12T23:14:52.947 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:52 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:14:52.474+0000 7fe5b1a697c0 -1 osd.7 147 log_to_monitors true 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: osdmap e150: 8 total, 7 up, 8 in 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi144.alilcy"]}]: dispatch 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: Upgrade: It appears safe to stop mds.foofs.smithi144.alilcy 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: from='osd.7 [v2:172.21.15.181:6824/3311184614,v1:172.21.15.181:6825/3311184614]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:14:53.589 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:53 smithi144 ceph-mon[67032]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:14:53.697 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:14:53.470+0000 7fe5a2ec1700 -1 osd.7 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: osdmap e150: 8 total, 7 up, 8 in 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi144.alilcy"]}]: dispatch 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: Upgrade: It appears safe to stop mds.foofs.smithi144.alilcy 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: from='osd.7 [v2:172.21.15.181:6824/3311184614,v1:172.21.15.181:6825/3311184614]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:14:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:53 smithi181 ceph-mon[52974]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-12T23:14:54.031 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:14:54.737 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (9m) 2m ago 18m 16.2M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (5m) 2m ago 18m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (5m) 7s ago 17m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 2m ago 18m - - 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (12m) 2m ago 12m 38.5M - 17.2.0 e1d6a67b021e 8e849a5cac52 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (12m) 7s ago 12m 15.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (8m) 2m ago 19m 509M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (7m) 7s ago 17m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (7m) 2m ago 20m 51.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (6m) 7s ago 17m 43.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (12m) 2m ago 12m 273M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (12m) 7s ago 12m 88.9M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:14:54.738 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (9m) 2m ago 18m 9000k - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (8m) 7s ago 17m 17.1M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (5m) 2m ago 17m 337M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (4m) 2m ago 16m 291M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (3m) 2m ago 16m 228M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (2m) 2m ago 15m 12.7M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (2m) 7s ago 15m 360M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (89s) 7s ago 15m 313M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (52s) 7s ago 14m 308M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (13s) 7s ago 14m 12.7M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:14:54.739 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (7m) 2m ago 17m 62.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:14:54.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:54 smithi144 ceph-mon[67032]: pgmap v306: 97 pgs: 18 active+undersized, 24 active+undersized+degraded, 55 active+clean; 232 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 1.9 MiB/s wr, 269 op/s; 71/432 objects degraded (16.435%) 2024-04-12T23:14:54.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:54 smithi144 ceph-mon[67032]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T23:14:54.739 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:54 smithi144 ceph-mon[67032]: osdmap e151: 8 total, 7 up, 8 in 2024-04-12T23:14:54.740 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:54 smithi144 ceph-mon[67032]: from='osd.7 [v2:172.21.15.181:6824/3311184614,v1:172.21.15.181:6825/3311184614]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:54.740 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:54 smithi144 ceph-mon[67032]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:54 smithi181 ceph-mon[52974]: pgmap v306: 97 pgs: 18 active+undersized, 24 active+undersized+degraded, 55 active+clean; 232 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 1.9 MiB/s wr, 269 op/s; 71/432 objects degraded (16.435%) 2024-04-12T23:14:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:54 smithi181 ceph-mon[52974]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-12T23:14:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:54 smithi181 ceph-mon[52974]: osdmap e151: 8 total, 7 up, 8 in 2024-04-12T23:14:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:54 smithi181 ceph-mon[52974]: from='osd.7 [v2:172.21.15.181:6824/3311184614,v1:172.21.15.181:6825/3311184614]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:54.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:54 smithi181 ceph-mon[52974]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-04-12T23:14:55.162 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:14:55.162 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 12 2024-04-12T23:14:55.163 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:14:55.164 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:14:55.539 INFO:teuthology.orchestra.run.smithi144.stdout: "mon", 2024-04-12T23:14:55.540 INFO:teuthology.orchestra.run.smithi144.stdout: "osd" 2024-04-12T23:14:55.540 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:14:55.540 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "14/23 daemons upgraded", 2024-04-12T23:14:55.540 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading mds daemons", 2024-04-12T23:14:55.540 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:14:55.540 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: osd.7 [v2:172.21.15.181:6824/3311184614,v1:172.21.15.181:6825/3311184614] boot 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: osdmap e152: 8 total, 8 up, 8 in 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: Upgrade: Updating mds.foofs.smithi144.alilcy 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: Deploying daemon mds.foofs.smithi144.alilcy on smithi144 2024-04-12T23:14:55.744 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:55.745 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/396539650' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:55.745 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 71/432 objects degraded (16.435%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:55.745 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:55 smithi144 ceph-mon[67032]: osdmap e153: 8 total, 8 up, 8 in 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: osd.7 [v2:172.21.15.181:6824/3311184614,v1:172.21.15.181:6825/3311184614] boot 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: osdmap e152: 8 total, 8 up, 8 in 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-12T23:14:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: Upgrade: Updating mds.foofs.smithi144.alilcy 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi144.alilcy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: Deploying daemon mds.foofs.smithi144.alilcy on smithi144 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/396539650' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 71/432 objects degraded (16.435%), 24 pgs degraded (PG_DEGRADED) 2024-04-12T23:14:55.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:55 smithi181 ceph-mon[52974]: osdmap e153: 8 total, 8 up, 8 in 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 68/432 objects degraded (15.741%), 23 pgs degraded 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 68/432 objects degraded (15.741%), 23 pgs degraded 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-04-12T23:14:55.956 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-04-12T23:14:55.957 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-04-12T23:14:55.958 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.a is active+undersized+degraded, acting [6,2] 2024-04-12T23:14:55.958 INFO:teuthology.orchestra.run.smithi144.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-04-12T23:14:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:56 smithi144 ceph-mon[67032]: pgmap v309: 97 pgs: 6 peering, 13 active+undersized, 23 active+undersized+degraded, 55 active+clean; 232 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 50 KiB/s wr, 36 op/s; 68/432 objects degraded (15.741%) 2024-04-12T23:14:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:56 smithi144 ceph-mon[67032]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:56 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3080966407' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:14:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:56.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:56 smithi144 ceph-mon[67032]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T23:14:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:56 smithi181 ceph-mon[52974]: pgmap v309: 97 pgs: 6 peering, 13 active+undersized, 23 active+undersized+degraded, 55 active+clean; 232 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 50 KiB/s wr, 36 op/s; 68/432 objects degraded (15.741%) 2024-04-12T23:14:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:56 smithi181 ceph-mon[52974]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:14:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:56 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3080966407' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:14:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:14:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:14:56.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:56 smithi181 ceph-mon[52974]: osdmap e154: 8 total, 8 up, 8 in 2024-04-12T23:14:58.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:57 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[67028]: 2024-04-12T23:14:57.802+0000 7ff524ca8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:14:58.274 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:14:57 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:14:57.824+0000 7f6726c58700 -1 bdev(0x55c4f7e36000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-04-12T23:14:58.274 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:14:57 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:14:57.824+0000 7f672ac60700 -1 bdev(0x55c4f7e36000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 4 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: pgmap v312: 97 pgs: 2 active+recovery_wait+degraded, 6 peering, 10 active+undersized, 18 active+undersized+degraded, 61 active+clean; 232 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 53 KiB/s wr, 1 op/s; 110/432 objects degraded (25.463%); 631 KiB/s, 0 objects/s recovering 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: Standby daemon mds.foofs.smithi181.gciyns assigned to filesystem foofs as rank 0 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T23:14:58.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:14:58 smithi181 ceph-mon[52974]: fsmap foofs:1/1 {0=foofs.smithi181.gciyns=up:replay} 2024-04-12T23:14:58.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: pgmap v312: 97 pgs: 2 active+recovery_wait+degraded, 6 peering, 10 active+undersized, 18 active+undersized+degraded, 61 active+clean; 232 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 53 KiB/s wr, 1 op/s; 110/432 objects degraded (25.463%); 631 KiB/s, 0 objects/s recovering 2024-04-12T23:14:58.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: osdmap e155: 8 total, 8 up, 8 in 2024-04-12T23:14:58.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T23:14:58.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:14:58.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: osdmap e156: 8 total, 8 up, 8 in 2024-04-12T23:14:58.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: Standby daemon mds.foofs.smithi181.gciyns assigned to filesystem foofs as rank 0 2024-04-12T23:14:58.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T23:14:58.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T23:14:58.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T23:14:58.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:14:58 smithi144 ceph-mon[67032]: fsmap foofs:1/1 {0=foofs.smithi181.gciyns=up:replay} 2024-04-12T23:15:00.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:00 smithi144 ceph-mon[67032]: pgmap v315: 97 pgs: 2 active+recovery_wait+degraded, 6 peering, 2 active+undersized+remapped, 87 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 53 KiB/s wr, 9 op/s; 59/429 objects degraded (13.753%); 3.6 MiB/s, 60 objects/s recovering 2024-04-12T23:15:00.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:00 smithi144 ceph-mon[67032]: reconnect by client.24325 172.21.15.181:0/2814205880 after 0.000999983 2024-04-12T23:15:00.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:00 smithi144 ceph-mon[67032]: reconnect by client.14552 172.21.15.144:0/3291878373 after 0.000999983 2024-04-12T23:15:00.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:00 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:reconnect 2024-04-12T23:15:00.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:00 smithi144 ceph-mon[67032]: fsmap foofs:1/1 {0=foofs.smithi181.gciyns=up:reconnect} 2024-04-12T23:15:00.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:00 smithi144 ceph-mon[67032]: Health check update: Degraded data redundancy: 59/429 objects degraded (13.753%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:15:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:00 smithi181 ceph-mon[52974]: pgmap v315: 97 pgs: 2 active+recovery_wait+degraded, 6 peering, 2 active+undersized+remapped, 87 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 53 KiB/s wr, 9 op/s; 59/429 objects degraded (13.753%); 3.6 MiB/s, 60 objects/s recovering 2024-04-12T23:15:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:00 smithi181 ceph-mon[52974]: reconnect by client.24325 172.21.15.181:0/2814205880 after 0.000999983 2024-04-12T23:15:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:00 smithi181 ceph-mon[52974]: reconnect by client.14552 172.21.15.144:0/3291878373 after 0.000999983 2024-04-12T23:15:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:00 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:reconnect 2024-04-12T23:15:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:00 smithi181 ceph-mon[52974]: fsmap foofs:1/1 {0=foofs.smithi181.gciyns=up:reconnect} 2024-04-12T23:15:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:00 smithi181 ceph-mon[52974]: Health check update: Degraded data redundancy: 59/429 objects degraded (13.753%), 2 pgs degraded (PG_DEGRADED) 2024-04-12T23:15:01.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:01 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:rejoin 2024-04-12T23:15:01.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:01 smithi144 ceph-mon[67032]: fsmap foofs:1/1 {0=foofs.smithi181.gciyns=up:rejoin} 2024-04-12T23:15:01.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:01 smithi144 ceph-mon[67032]: daemon mds.foofs.smithi181.gciyns is now active in filesystem foofs as rank 0 2024-04-12T23:15:01.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:01 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:rejoin 2024-04-12T23:15:01.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:01 smithi181 ceph-mon[52974]: fsmap foofs:1/1 {0=foofs.smithi181.gciyns=up:rejoin} 2024-04-12T23:15:01.919 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:01 smithi181 ceph-mon[52974]: daemon mds.foofs.smithi181.gciyns is now active in filesystem foofs as rank 0 2024-04-12T23:15:02.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:02 smithi144 ceph-mon[67032]: pgmap v316: 97 pgs: 2 active+undersized+remapped, 95 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 36 KiB/s wr, 8 op/s; 2.5 MiB/s, 63 objects/s recovering 2024-04-12T23:15:02.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:02 smithi144 ceph-mon[67032]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 59/429 objects degraded (13.753%), 2 pgs degraded) 2024-04-12T23:15:02.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:02 smithi144 ceph-mon[67032]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T23:15:02.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:02 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:active 2024-04-12T23:15:02.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:02 smithi144 ceph-mon[67032]: fsmap foofs:1 {0=foofs.smithi181.gciyns=up:active} 2024-04-12T23:15:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:02 smithi181 ceph-mon[52974]: pgmap v316: 97 pgs: 2 active+undersized+remapped, 95 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 36 KiB/s wr, 8 op/s; 2.5 MiB/s, 63 objects/s recovering 2024-04-12T23:15:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:02 smithi181 ceph-mon[52974]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 59/429 objects degraded (13.753%), 2 pgs degraded) 2024-04-12T23:15:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:02 smithi181 ceph-mon[52974]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T23:15:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:02 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.181:6832/1277135509,v1:172.21.15.181:6833/1277135509] up:active 2024-04-12T23:15:02.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:02 smithi181 ceph-mon[52974]: fsmap foofs:1 {0=foofs.smithi181.gciyns=up:active} 2024-04-12T23:15:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:04 smithi144 ceph-mon[67032]: pgmap v317: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 31 KiB/s wr, 13 op/s; 2.1 MiB/s, 53 objects/s recovering 2024-04-12T23:15:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:04 smithi181 ceph-mon[52974]: pgmap v317: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 31 KiB/s wr, 13 op/s; 2.1 MiB/s, 53 objects/s recovering 2024-04-12T23:15:06.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:06 smithi144 ceph-mon[67032]: pgmap v318: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 255 B/s wr, 13 op/s; 1.5 MiB/s, 46 objects/s recovering 2024-04-12T23:15:06.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:06 smithi181 ceph-mon[52974]: pgmap v318: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 255 B/s wr, 13 op/s; 1.5 MiB/s, 46 objects/s recovering 2024-04-12T23:15:08.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:08.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:08.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:08.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T23:15:08.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:boot 2024-04-12T23:15:08.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: fsmap foofs:1 {0=foofs.smithi181.gciyns=up:active} 1 up:standby 2024-04-12T23:15:08.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:07 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:boot 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: fsmap foofs:1 {0=foofs.smithi181.gciyns=up:active} 1 up:standby 2024-04-12T23:15:08.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:07 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:15:09.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:08 smithi144 ceph-mon[67032]: pgmap v319: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s; 830 KiB/s, 34 objects/s recovering 2024-04-12T23:15:09.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:08 smithi181 ceph-mon[52974]: pgmap v319: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s; 830 KiB/s, 34 objects/s recovering 2024-04-12T23:15:11.000 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:10 smithi181 ceph-mon[52974]: pgmap v320: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.5 KiB/s wr, 12 op/s; 711 KiB/s, 29 objects/s recovering 2024-04-12T23:15:11.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:10 smithi144 ceph-mon[67032]: pgmap v320: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.5 KiB/s wr, 12 op/s; 711 KiB/s, 29 objects/s recovering 2024-04-12T23:15:12.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:12 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:12.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:12 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:12.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:12 smithi181 ceph-mon[52974]: pgmap v321: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 KiB/s wr, 9 op/s; 0 B/s, 10 objects/s recovering 2024-04-12T23:15:12.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:12 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:12.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:12 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:12.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:12 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:12.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:12 smithi144 ceph-mon[67032]: pgmap v321: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 KiB/s wr, 9 op/s; 0 B/s, 10 objects/s recovering 2024-04-12T23:15:12.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:12 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:14.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:14 smithi144 ceph-mon[67032]: pgmap v322: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-04-12T23:15:14.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:14.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:14 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:14 smithi181 ceph-mon[52974]: pgmap v322: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-04-12T23:15:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:14.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:14 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:15.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:15.955 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:15.955 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:16.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:16 smithi144 ceph-mon[67032]: pgmap v323: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 7.0 KiB/s wr, 4 op/s 2024-04-12T23:15:16.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:16 smithi181 ceph-mon[52974]: pgmap v323: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s rd, 7.0 KiB/s wr, 4 op/s 2024-04-12T23:15:18.610 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:18 smithi144 ceph-mon[67032]: pgmap v324: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 340 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-04-12T23:15:18.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:18 smithi181 ceph-mon[52974]: pgmap v324: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 340 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-04-12T23:15:20.606 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: pgmap v325: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: Detected new or changed devices on smithi144 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: pgmap v326: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 201 B/s wr, 0 op/s 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: pgmap v327: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 251 B/s rd, 251 B/s wr, 0 op/s 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.607 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.608 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.608 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi181.gciyns"]}]: dispatch 2024-04-12T23:15:20.608 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:20 smithi181 ceph-mon[52974]: Upgrade: It appears safe to stop mds.foofs.smithi181.gciyns 2024-04-12T23:15:20.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: pgmap v325: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2024-04-12T23:15:20.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: Detected new or changed devices on smithi144 2024-04-12T23:15:20.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: pgmap v326: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 201 B/s rd, 201 B/s wr, 0 op/s 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: pgmap v327: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 251 B/s rd, 251 B/s wr, 0 op/s 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi181.gciyns"]}]: dispatch 2024-04-12T23:15:20.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:20 smithi144 ceph-mon[67032]: Upgrade: It appears safe to stop mds.foofs.smithi181.gciyns 2024-04-12T23:15:22.508 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:22 smithi181 ceph-mon[52974]: Upgrade: Updating mds.foofs.smithi181.gciyns 2024-04-12T23:15:22.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:22 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:22.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:22 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:15:22.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:22 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:22.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:22 smithi181 ceph-mon[52974]: Deploying daemon mds.foofs.smithi181.gciyns on smithi181 2024-04-12T23:15:22.509 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:22 smithi181 ceph-mon[52974]: pgmap v328: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 251 B/s rd, 251 B/s wr, 0 op/s 2024-04-12T23:15:22.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:22 smithi144 ceph-mon[67032]: Upgrade: Updating mds.foofs.smithi181.gciyns 2024-04-12T23:15:22.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:22 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:22.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:22 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi181.gciyns", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-12T23:15:22.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:22 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:22.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:22 smithi144 ceph-mon[67032]: Deploying daemon mds.foofs.smithi181.gciyns on smithi181 2024-04-12T23:15:22.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:22 smithi144 ceph-mon[67032]: pgmap v328: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 251 B/s rd, 251 B/s wr, 0 op/s 2024-04-12T23:15:23.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T23:15:23.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:15:23.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T23:15:23.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: Standby daemon mds.foofs.smithi144.alilcy assigned to filesystem foofs as rank 0 2024-04-12T23:15:23.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T23:15:23.699 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T23:15:23.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T23:15:23.700 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:23 smithi181 ceph-mon[52974]: fsmap foofs:1/1 {0=foofs.smithi144.alilcy=up:replay} 2024-04-12T23:15:23.773 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:15:23.423+0000 7f672c463700 -1 bdev(0x55c4f7e36000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 5 2024-04-12T23:15:23.773 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:15:23.423+0000 7f6726c58700 -1 bdev(0x55c4f7e36000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 6 2024-04-12T23:15:23.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[67028]: 2024-04-12T23:15:23.400+0000 7ff524ca8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: osdmap e157: 8 total, 8 up, 8 in 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: Standby daemon mds.foofs.smithi144.alilcy assigned to filesystem foofs as rank 0 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-04-12T23:15:23.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:23 smithi144 ceph-mon[67032]: fsmap foofs:1/1 {0=foofs.smithi144.alilcy=up:replay} 2024-04-12T23:15:24.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:24 smithi181 ceph-mon[52974]: pgmap v330: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1 op/s 2024-04-12T23:15:24.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:24 smithi144 ceph-mon[67032]: pgmap v330: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 1 op/s 2024-04-12T23:15:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:25 smithi181 ceph-mon[52974]: reconnect by client.14552 172.21.15.144:0/3291878373 after 0.000999983 2024-04-12T23:15:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:25 smithi181 ceph-mon[52974]: reconnect by client.24325 172.21.15.181:0/2814205880 after 0.000999983 2024-04-12T23:15:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:25 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:reconnect 2024-04-12T23:15:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:25 smithi181 ceph-mon[52974]: fsmap foofs:1/1 {0=foofs.smithi144.alilcy=up:reconnect} 2024-04-12T23:15:25.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:25 smithi144 ceph-mon[67032]: reconnect by client.14552 172.21.15.144:0/3291878373 after 0.000999983 2024-04-12T23:15:25.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:25 smithi144 ceph-mon[67032]: reconnect by client.24325 172.21.15.181:0/2814205880 after 0.000999983 2024-04-12T23:15:25.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:25 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:reconnect 2024-04-12T23:15:25.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:25 smithi144 ceph-mon[67032]: fsmap foofs:1/1 {0=foofs.smithi144.alilcy=up:reconnect} 2024-04-12T23:15:26.306 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:15:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:26 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:rejoin 2024-04-12T23:15:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:26 smithi144 ceph-mon[67032]: fsmap foofs:1/1 {0=foofs.smithi144.alilcy=up:rejoin} 2024-04-12T23:15:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:26 smithi144 ceph-mon[67032]: daemon mds.foofs.smithi144.alilcy is now active in filesystem foofs as rank 0 2024-04-12T23:15:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:26 smithi144 ceph-mon[67032]: pgmap v331: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2 op/s 2024-04-12T23:15:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:26.872 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:26 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:rejoin 2024-04-12T23:15:26.872 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:26 smithi181 ceph-mon[52974]: fsmap foofs:1/1 {0=foofs.smithi144.alilcy=up:rejoin} 2024-04-12T23:15:26.872 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:26 smithi181 ceph-mon[52974]: daemon mds.foofs.smithi144.alilcy is now active in filesystem foofs as rank 0 2024-04-12T23:15:26.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:26 smithi181 ceph-mon[52974]: pgmap v331: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2 op/s 2024-04-12T23:15:26.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:26.873 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (10m) 13s ago 19m 24.0M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (6m) 13s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (6m) 40s ago 18m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 13s ago 18m - - 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (20s) 13s ago 13m 11.2M - 19.0.0-2053-gb277feb2 668895944f0e 8982b4407488 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (13m) 40s ago 13m 15.4M - 17.2.0 e1d6a67b021e 9724fc5b1559 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (9m) 13s ago 20m 513M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:15:26.974 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (8m) 40s ago 18m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (7m) 13s ago 20m 60.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (7m) 40s ago 18m 43.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (13m) 13s ago 13m 267M - 4.0 e1d6a67b021e 5cc36b6e297a 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (12m) 40s ago 12m 88.9M - 4.0 e1d6a67b021e 311f4aa1b4c6 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (9m) 13s ago 18m 17.2M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (9m) 40s ago 18m 17.1M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (5m) 13s ago 17m 424M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (4m) 13s ago 17m 380M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (4m) 13s ago 16m 390M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (3m) 13s ago 16m 417M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (2m) 40s ago 15m 360M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (2m) 40s ago 15m 313M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (84s) 40s ago 15m 308M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (45s) 40s ago 14m 12.7M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:15:26.975 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (8m) 13s ago 17m 63.8M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:15:27.380 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 1 2024-04-12T23:15:27.381 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:27.381 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:15:27.381 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 13 2024-04-12T23:15:27.381 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:15:27.381 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:15:27.723 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:27 smithi181 ceph-mon[52974]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:27.723 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:27 smithi181 ceph-mon[52974]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T23:15:27.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:27 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:active 2024-04-12T23:15:27.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:27 smithi181 ceph-mon[52974]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 2024-04-12T23:15:27.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:27 smithi181 ceph-mon[52974]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:27.724 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:27 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3587514185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:27.724 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "mon", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "osd" 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "15/23 daemons upgraded", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading mds daemons", 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:15:27.725 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:15:27.726 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:27 smithi144 ceph-mon[67032]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:27.726 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:27 smithi144 ceph-mon[67032]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-12T23:15:27.726 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:27 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160] up:active 2024-04-12T23:15:27.726 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:27 smithi144 ceph-mon[67032]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 2024-04-12T23:15:27.726 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:27 smithi144 ceph-mon[67032]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:27.726 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:27 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3587514185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:28.129 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s); insufficient standby MDS daemons available 2024-04-12T23:15:28.129 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:15:28.129 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:15:28.129 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-04-12T23:15:28.129 INFO:teuthology.orchestra.run.smithi144.stdout: have 0; want 1 more 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: pgmap v332: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 127 B/s wr, 7 op/s 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:28.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:28 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1806926690' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: pgmap v332: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 127 B/s wr, 7 op/s 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:28.881 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:28 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1806926690' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:15:29.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:29 smithi144 ceph-mon[67032]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T23:15:29.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:29 smithi144 ceph-mon[67032]: mds.? [v2:172.21.15.181:6832/2818024163,v1:172.21.15.181:6833/2818024163] up:boot 2024-04-12T23:15:29.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:29 smithi144 ceph-mon[67032]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:15:29.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:29 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:15:29.850 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:29 smithi181 ceph-mon[52974]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-12T23:15:29.850 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:29 smithi181 ceph-mon[52974]: mds.? [v2:172.21.15.181:6832/2818024163,v1:172.21.15.181:6833/2818024163] up:boot 2024-04-12T23:15:29.850 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:29 smithi181 ceph-mon[52974]: fsmap foofs:1 {0=foofs.smithi144.alilcy=up:active} 1 up:standby 2024-04-12T23:15:29.850 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:29 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:15:30.636 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:30 smithi181 ceph-mon[52974]: pgmap v333: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 10 op/s 2024-04-12T23:15:30.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:30 smithi144 ceph-mon[67032]: pgmap v333: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 10 op/s 2024-04-12T23:15:32.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:32.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:32.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:32 smithi181 ceph-mon[52974]: pgmap v334: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 11 op/s 2024-04-12T23:15:32.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:32.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:32.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:32 smithi144 ceph-mon[67032]: pgmap v334: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 11 op/s 2024-04-12T23:15:33.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:33.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:33.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:33.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:34.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:34 smithi181 ceph-mon[52974]: pgmap v335: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.6 KiB/s wr, 14 op/s 2024-04-12T23:15:34.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:34 smithi144 ceph-mon[67032]: pgmap v335: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.6 KiB/s wr, 14 op/s 2024-04-12T23:15:36.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: pgmap v336: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-04-12T23:15:36.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:36.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.alilcy"}]': finished 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.gciyns"}]': finished 2024-04-12T23:15:36.774 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.775 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:36 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: pgmap v336: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-04-12T23:15:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:15:36.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.alilcy"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi144.alilcy"}]': finished 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.gciyns"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi181.gciyns"}]': finished 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:36.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:36 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: Detected new or changed devices on smithi181 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: pgmap v337: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.5 KiB/s wr, 13 op/s 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: pgmap v338: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 10 KiB/s wr, 11 op/s 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all mds 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all rgw 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all rbd-mirror 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all ceph-exporter 2024-04-12T23:15:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:37 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all iscsi 2024-04-12T23:15:38.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: Detected new or changed devices on smithi181 2024-04-12T23:15:38.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: pgmap v337: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.5 KiB/s wr, 13 op/s 2024-04-12T23:15:38.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: pgmap v338: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 10 KiB/s wr, 11 op/s 2024-04-12T23:15:38.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all mds 2024-04-12T23:15:38.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all rgw 2024-04-12T23:15:38.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all rbd-mirror 2024-04-12T23:15:38.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all ceph-exporter 2024-04-12T23:15:38.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:37 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all iscsi 2024-04-12T23:15:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: Upgrade: Updating nfs.foo.0.0.smithi144.mjwmkt (1/2) 2024-04-12T23:15:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt 2024-04-12T23:15:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:15:38.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:38 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:39.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: Upgrade: Updating nfs.foo.0.0.smithi144.mjwmkt (1/2) 2024-04-12T23:15:39.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:39.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt 2024-04-12T23:15:39.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:15:39.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi144.mjwmkt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:15:39.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:38 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:39 smithi144 ceph-mon[67032]: Rados config object exists: conf-nfs.foo 2024-04-12T23:15:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:39 smithi144 ceph-mon[67032]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt-rgw 2024-04-12T23:15:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:39 smithi144 ceph-mon[67032]: Bind address in nfs.foo.0.0.smithi144.mjwmkt's ganesha conf is defaulting to empty 2024-04-12T23:15:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:39 smithi144 ceph-mon[67032]: Deploying daemon nfs.foo.0.0.smithi144.mjwmkt on smithi144 2024-04-12T23:15:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:39 smithi144 ceph-mon[67032]: pgmap v339: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 10 KiB/s wr, 5 op/s 2024-04-12T23:15:40.031 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:39 smithi181 ceph-mon[52974]: Rados config object exists: conf-nfs.foo 2024-04-12T23:15:40.031 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:39 smithi181 ceph-mon[52974]: Creating key for client.nfs.foo.0.0.smithi144.mjwmkt-rgw 2024-04-12T23:15:40.031 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:39 smithi181 ceph-mon[52974]: Bind address in nfs.foo.0.0.smithi144.mjwmkt's ganesha conf is defaulting to empty 2024-04-12T23:15:40.031 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:39 smithi181 ceph-mon[52974]: Deploying daemon nfs.foo.0.0.smithi144.mjwmkt on smithi144 2024-04-12T23:15:40.031 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:39 smithi181 ceph-mon[52974]: pgmap v339: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 10 KiB/s wr, 5 op/s 2024-04-12T23:15:41.928 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:41 smithi144 ceph-mon[67032]: pgmap v340: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 247 B/s rd, 10 KiB/s wr, 4 op/s 2024-04-12T23:15:41.928 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:41.928 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:42.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:41 smithi181 ceph-mon[52974]: pgmap v340: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 247 B/s rd, 10 KiB/s wr, 4 op/s 2024-04-12T23:15:42.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:42.078 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:42.978 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:42 smithi181 ceph-mon[52974]: Evicting (and blocklisting) client session 14552 (172.21.15.144:0/3291878373) 2024-04-12T23:15:42.978 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:42 smithi181 ceph-mon[52974]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/3291878373"}]: dispatch 2024-04-12T23:15:42.978 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:42.978 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:42 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:43.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:42 smithi144 ceph-mon[67032]: Evicting (and blocklisting) client session 14552 (172.21.15.144:0/3291878373) 2024-04-12T23:15:43.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:42 smithi144 ceph-mon[67032]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/3291878373"}]: dispatch 2024-04-12T23:15:43.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:43.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:42 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:44.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:43 smithi144 ceph-mon[67032]: pgmap v341: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 618 B/s wr, 2 op/s 2024-04-12T23:15:44.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:43 smithi144 ceph-mon[67032]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/3291878373"}]': finished 2024-04-12T23:15:44.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:43 smithi144 ceph-mon[67032]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T23:15:44.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:43 smithi181 ceph-mon[52974]: pgmap v341: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 618 B/s wr, 2 op/s 2024-04-12T23:15:44.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:43 smithi181 ceph-mon[52974]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.144:0/3291878373"}]': finished 2024-04-12T23:15:44.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:43 smithi181 ceph-mon[52974]: osdmap e158: 8 total, 8 up, 8 in 2024-04-12T23:15:45.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: pgmap v343: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 895 B/s wr, 2 op/s 2024-04-12T23:15:45.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Upgrade: Updating nfs.foo.1.0.smithi181.dgxqfb (2/2) 2024-04-12T23:15:45.149 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Rados config object exists: conf-nfs.foo 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb-rgw 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Bind address in nfs.foo.1.0.smithi181.dgxqfb's ganesha conf is defaulting to empty 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:45.150 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:44 smithi181 ceph-mon[52974]: Deploying daemon nfs.foo.1.0.smithi181.dgxqfb on smithi181 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: pgmap v343: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 895 B/s wr, 2 op/s 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Upgrade: Updating nfs.foo.1.0.smithi181.dgxqfb (2/2) 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-04-12T23:15:45.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Rados config object exists: conf-nfs.foo 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Creating key for client.nfs.foo.1.0.smithi181.dgxqfb-rgw 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi181.dgxqfb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Bind address in nfs.foo.1.0.smithi181.dgxqfb's ganesha conf is defaulting to empty 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:45.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:44 smithi144 ceph-mon[67032]: Deploying daemon nfs.foo.1.0.smithi181.dgxqfb on smithi181 2024-04-12T23:15:47.119 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:46 smithi181 ceph-mon[52974]: pgmap v344: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 921 B/s wr, 3 op/s 2024-04-12T23:15:47.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:46 smithi144 ceph-mon[67032]: pgmap v344: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 921 B/s wr, 3 op/s 2024-04-12T23:15:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:48.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:48.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:48.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:48.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:49 smithi181 ceph-mon[52974]: pgmap v345: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-12T23:15:50.016 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:49 smithi144 ceph-mon[67032]: pgmap v345: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-04-12T23:15:50.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:50 smithi144 ceph-mon[67032]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2814205880"}]: dispatch 2024-04-12T23:15:50.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:50.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:50.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:50 smithi181 ceph-mon[52974]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2814205880"}]: dispatch 2024-04-12T23:15:50.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:50.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:51.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:51 smithi181 ceph-mon[52974]: pgmap v346: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-12T23:15:51.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:51 smithi181 ceph-mon[52974]: Evicting (and blocklisting) client session 24325 (172.21.15.181:0/2814205880) 2024-04-12T23:15:51.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:51 smithi181 ceph-mon[52974]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2814205880"}]': finished 2024-04-12T23:15:51.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:51 smithi181 ceph-mon[52974]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T23:15:51.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:51 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:52.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:51 smithi144 ceph-mon[67032]: pgmap v346: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-12T23:15:52.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:51 smithi144 ceph-mon[67032]: Evicting (and blocklisting) client session 24325 (172.21.15.181:0/2814205880) 2024-04-12T23:15:52.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:51 smithi144 ceph-mon[67032]: from='mds.? [v2:172.21.15.144:6834/3850637160,v1:172.21.15.144:6835/3850637160]' entity='mds.foofs.smithi144.alilcy' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.181:0/2814205880"}]': finished 2024-04-12T23:15:52.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:51 smithi144 ceph-mon[67032]: osdmap e159: 8 total, 8 up, 8 in 2024-04-12T23:15:52.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:51 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:52.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:52.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:52.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:52 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:52 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:53 smithi181 ceph-mon[52974]: pgmap v348: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2024-04-12T23:15:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:53 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:53 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:53 smithi144 ceph-mon[67032]: pgmap v348: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.3 KiB/s wr, 7 op/s 2024-04-12T23:15:53.904 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:53.905 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:53 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:55.875 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:55 smithi144 ceph-mon[67032]: pgmap v349: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-04-12T23:15:55.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:55.876 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:55 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:55 smithi181 ceph-mon[52974]: pgmap v349: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-04-12T23:15:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:55.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:55 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: pgmap v350: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.7 KiB/s wr, 4 op/s 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: pgmap v351: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: pgmap v352: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi144.mjwmkt"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi144.mjwmkt"}]': finished 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.dgxqfb"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.dgxqfb"}]': finished 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:57.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:57 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:58.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: pgmap v350: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.7 KiB/s wr, 4 op/s 2024-04-12T23:15:58.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: pgmap v351: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: pgmap v352: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi144.mjwmkt"}]: dispatch 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi144.mjwmkt"}]': finished 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.dgxqfb"}]: dispatch 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi181.dgxqfb"}]': finished 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:15:58.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:57 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:15:58.486 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:15:58.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:58 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all nfs 2024-04-12T23:15:58.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:58 smithi181 ceph-mon[52974]: Upgrade: Setting container_image for all nvmeof 2024-04-12T23:15:59.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:58 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all nfs 2024-04-12T23:15:59.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:58 smithi144 ceph-mon[67032]: Upgrade: Setting container_image for all nvmeof 2024-04-12T23:15:59.232 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:15:59.232 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (10m) 7s ago 19m 23.7M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:15:59.232 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (6m) 7s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:15:59.232 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (6m) 8s ago 18m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:15:59.232 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 7s ago 19m - - 2024-04-12T23:15:59.232 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (53s) 7s ago 13m 35.4M - 19.0.0-2053-gb277feb2 668895944f0e 8982b4407488 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (31s) 8s ago 13m 11.4M - 19.0.0-2053-gb277feb2 668895944f0e 04579505b4e8 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (9m) 7s ago 20m 514M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (8m) 8s ago 18m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (8m) 7s ago 21m 62.2M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (7m) 8s ago 18m 45.4M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (17s) 7s ago 13m 61.4M - 5.7 668895944f0e d28a74ff9bef 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (12s) 8s ago 13m 19.2M - 5.7 668895944f0e aa8f9b410175 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (10m) 7s ago 19m 17.7M - 1.3.1 1dbe0e931976 512e9528a3cf 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (9m) 8s ago 18m 19.1M - 1.3.1 1dbe0e931976 3ccb01df7e6c 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (6m) 7s ago 18m 425M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:15:59.233 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (5m) 7s ago 17m 382M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (4m) 7s ago 17m 393M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (3m) 7s ago 16m 420M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (3m) 8s ago 16m 371M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (2m) 8s ago 16m 325M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (116s) 8s ago 15m 309M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (77s) 8s ago 15m 297M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:15:59.234 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (8m) 7s ago 18m 68.9M - 2.33.4 514e6a882f6e 866d13cbd126 2024-04-12T23:15:59.664 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:15:59.664 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:15:59.664 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:15:59.664 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:59.664 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:15:59.665 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:15:59.878 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:59 smithi144 ceph-mon[67032]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:59.878 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:59 smithi144 ceph-mon[67032]: Upgrade: Updating node-exporter.smithi144 (1/2) 2024-04-12T23:15:59.879 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:59 smithi144 ceph-mon[67032]: Deploying daemon node-exporter.smithi144 on smithi144 2024-04-12T23:15:59.879 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:15:59 smithi144 ceph-mon[67032]: pgmap v353: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-12T23:15:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:59 smithi181 ceph-mon[52974]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:15:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:59 smithi181 ceph-mon[52974]: Upgrade: Updating node-exporter.smithi144 (1/2) 2024-04-12T23:15:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:59 smithi181 ceph-mon[52974]: Deploying daemon node-exporter.smithi144 on smithi144 2024-04-12T23:15:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:15:59 smithi181 ceph-mon[52974]: pgmap v353: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "nfs", 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "mds", 2024-04-12T23:16:00.002 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: "osd", 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "18/23 daemons upgraded", 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:16:00.003 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:16:00.432 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:16:00.433 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:16:00.433 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:16:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:00 smithi181 ceph-mon[52974]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:00 smithi181 ceph-mon[52974]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:00 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/422197454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:00 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1772945954' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:16:01.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:00 smithi144 ceph-mon[67032]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:01.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:00 smithi144 ceph-mon[67032]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:01.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:00 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/422197454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:01.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:00 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1772945954' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:16:01.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:01 smithi181 ceph-mon[52974]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:01.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:01 smithi181 ceph-mon[52974]: pgmap v354: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-04-12T23:16:02.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:01 smithi144 ceph-mon[67032]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:02.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:01 smithi144 ceph-mon[67032]: pgmap v354: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-04-12T23:16:03.751 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:03 smithi144 ceph-mon[67032]: pgmap v355: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 231 B/s rd, 0 op/s 2024-04-12T23:16:03.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:03 smithi181 ceph-mon[52974]: pgmap v355: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 231 B/s rd, 0 op/s 2024-04-12T23:16:05.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:05.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:05.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:04 smithi181 ceph-mon[52974]: pgmap v356: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 op/s 2024-04-12T23:16:05.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:05.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:05.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:04 smithi144 ceph-mon[67032]: pgmap v356: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 op/s 2024-04-12T23:16:06.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:05 smithi181 ceph-mon[52974]: Upgrade: Updating node-exporter.smithi181 (2/2) 2024-04-12T23:16:06.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:05 smithi181 ceph-mon[52974]: Deploying daemon node-exporter.smithi181 on smithi181 2024-04-12T23:16:06.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:05 smithi144 ceph-mon[67032]: Upgrade: Updating node-exporter.smithi181 (2/2) 2024-04-12T23:16:06.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:05 smithi144 ceph-mon[67032]: Deploying daemon node-exporter.smithi181 on smithi181 2024-04-12T23:16:06.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:06 smithi181 ceph-mon[52974]: pgmap v357: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:16:07.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:06 smithi144 ceph-mon[67032]: pgmap v357: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:16:09.940 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:09 smithi181 ceph-mon[52974]: pgmap v358: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:09 smithi144 ceph-mon[67032]: pgmap v358: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:11.519 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:11.519 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:11.519 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:11.519 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:11 smithi181 ceph-mon[52974]: pgmap v359: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:11.519 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:11.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:11.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:11.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:11.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:11 smithi144 ceph-mon[67032]: pgmap v359: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:11.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:13 smithi181 ceph-mon[52974]: pgmap v360: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:13.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:13 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:14.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:13 smithi144 ceph-mon[67032]: pgmap v360: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:14.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:14.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:13 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:15 smithi181 ceph-mon[52974]: pgmap v361: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:15 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:15 smithi144 ceph-mon[67032]: pgmap v361: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:15 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:17.446 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:17.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:17 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:17.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:17 smithi181 ceph-mon[52974]: pgmap v362: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:17.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:17.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:17 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:17.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:17 smithi144 ceph-mon[67032]: pgmap v362: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:19.232 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:19.232 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:18 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:19.233 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:18 smithi144 ceph-mon[67032]: pgmap v363: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:18 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:19.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:18 smithi181 ceph-mon[52974]: pgmap v363: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:21.008 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.008 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.008 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: pgmap v364: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: pgmap v365: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.009 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.010 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:20 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: pgmap v364: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 180 B/s rd, 0 op/s 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: pgmap v365: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:21.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:21.199 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:20 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:22.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:21 smithi144 ceph-mon[67032]: Upgrade: Updating prometheus.smithi144 2024-04-12T23:16:22.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:21 smithi181 ceph-mon[52974]: Upgrade: Updating prometheus.smithi144 2024-04-12T23:16:23.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:22 smithi144 ceph-mon[67032]: Deploying daemon prometheus.smithi144 on smithi144 2024-04-12T23:16:23.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:22 smithi144 ceph-mon[67032]: pgmap v366: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:23.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:22 smithi181 ceph-mon[52974]: Deploying daemon prometheus.smithi144 on smithi144 2024-04-12T23:16:23.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:22 smithi181 ceph-mon[52974]: pgmap v366: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:25.267 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:24 smithi144 ceph-mon[67032]: pgmap v367: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:25.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:24 smithi181 ceph-mon[52974]: pgmap v367: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:27.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:26 smithi144 ceph-mon[67032]: pgmap v368: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:27.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:27.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:26 smithi181 ceph-mon[52974]: pgmap v368: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:27.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:28.402 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.402 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:28.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.403 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:28 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:28.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:28 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:29.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:29 smithi181 ceph-mon[52974]: pgmap v369: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:30.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:29 smithi144 ceph-mon[67032]: pgmap v369: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 218 B/s rd, 0 op/s 2024-04-12T23:16:30.811 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:16:31.505 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (11m) 16s ago 20m 23.7M - 0.23.0 ba2b418f427c e8d0402accfa 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (7m) 16s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (7m) 18s ago 19m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 error 16s ago 19m - - 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (85s) 16s ago 14m 35.7M - 19.0.0-2053-gb277feb2 668895944f0e 8982b4407488 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (63s) 18s ago 14m 11.5M - 19.0.0-2053-gb277feb2 668895944f0e 04579505b4e8 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (10m) 16s ago 21m 515M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (9m) 18s ago 19m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (9m) 16s ago 21m 62.7M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (8m) 18s ago 19m 45.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (50s) 16s ago 14m 63.6M - 5.7 668895944f0e d28a74ff9bef 2024-04-12T23:16:31.506 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (44s) 18s ago 13m 61.7M - 5.7 668895944f0e aa8f9b410175 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (28s) 16s ago 19m 3586k - 1.5.0 0da6a335fe13 92260f489e83 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (21s) 18s ago 19m 5486k - 1.5.0 0da6a335fe13 7804af9122fe 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (6m) 16s ago 18m 426M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (6m) 16s ago 18m 383M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (5m) 16s ago 17m 393M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (4m) 16s ago 17m 421M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (3m) 18s ago 17m 371M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (3m) 18s ago 16m 326M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (2m) 18s ago 16m 309M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (110s) 18s ago 15m 297M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:16:31.507 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 starting - - - - 2024-04-12T23:16:31.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:31 smithi144 ceph-mon[67032]: pgmap v370: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:16:31.906 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:16:31.907 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:16:31.907 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:16:31.907 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:16:31.907 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T23:16:31.907 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:16:31.907 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:16:31.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:31 smithi181 ceph-mon[52974]: pgmap v370: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:16:32.245 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:16:32.245 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:16:32.245 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:16:32.245 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "nfs", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "mds", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "osd", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "18/23 daemons upgraded", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:16:32.246 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:16:32.654 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:16:32.654 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:16:32.654 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3130435014' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:32.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:32 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1954157130' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3130435014' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:33.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:32 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1954157130' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:16:33.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:33 smithi181 ceph-mon[52974]: pgmap v371: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:33.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:33 smithi181 ceph-mon[52974]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:33.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:33.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:33 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:33.993 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:33 smithi144 ceph-mon[67032]: pgmap v371: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:33.993 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:33 smithi144 ceph-mon[67032]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:16:33.993 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:33.993 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:33 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:35.942 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:35 smithi144 ceph-mon[67032]: pgmap v372: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:35.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:35 smithi181 ceph-mon[52974]: pgmap v372: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: pgmap v373: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.769 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.770 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:37 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: pgmap v373: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:16:37.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:37.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:37 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:38.946 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:38 smithi144 ceph-mon[67032]: pgmap v374: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 189 B/s rd, 0 op/s 2024-04-12T23:16:38.947 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:38 smithi144 ceph-mon[67032]: pgmap v375: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:38.947 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:38 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:16:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:38 smithi181 ceph-mon[52974]: pgmap v374: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 189 B/s rd, 0 op/s 2024-04-12T23:16:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:38 smithi181 ceph-mon[52974]: pgmap v375: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:38.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:38 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-12T23:16:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:39 smithi144 ceph-mon[67032]: Upgrade: Updating alertmanager.smithi144 2024-04-12T23:16:40.014 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:39 smithi144 ceph-mon[67032]: Deploying daemon alertmanager.smithi144 on smithi144 2024-04-12T23:16:40.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:39 smithi181 ceph-mon[52974]: Upgrade: Updating alertmanager.smithi144 2024-04-12T23:16:40.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:39 smithi181 ceph-mon[52974]: Deploying daemon alertmanager.smithi144 on smithi144 2024-04-12T23:16:41.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:40 smithi144 ceph-mon[67032]: pgmap v376: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:41.099 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:40 smithi181 ceph-mon[52974]: pgmap v376: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:42.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:42.114 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:42.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:42 smithi181 ceph-mon[52974]: pgmap v377: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:43.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:42 smithi144 ceph-mon[67032]: pgmap v377: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:45.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:44 smithi144 ceph-mon[67032]: pgmap v378: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:45.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:45.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:45.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:44 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:45.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:44 smithi181 ceph-mon[52974]: pgmap v378: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 232 B/s rd, 0 op/s 2024-04-12T23:16:45.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:45.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:45.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:44 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:47.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:46 smithi144 ceph-mon[67032]: pgmap v379: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 116 B/s rd, 0 op/s 2024-04-12T23:16:47.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:46 smithi181 ceph-mon[52974]: pgmap v379: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 116 B/s rd, 0 op/s 2024-04-12T23:16:49.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:48 smithi144 ceph-mon[67032]: pgmap v380: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:16:49.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:49.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:48 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:49.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:48 smithi181 ceph-mon[52974]: pgmap v380: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:16:49.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:49.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:48 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:50.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:50 smithi144 ceph-mon[67032]: pgmap v381: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:50.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:50.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:50 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:50.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:50 smithi181 ceph-mon[52974]: pgmap v381: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:50.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:50.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:50 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:52.637 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:52 smithi144 ceph-mon[67032]: pgmap v382: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:52.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:52 smithi181 ceph-mon[52974]: pgmap v382: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:16:54.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: pgmap v383: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:54.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: pgmap v384: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 199 B/s rd, 0 op/s 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: pgmap v385: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.274 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.275 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:54 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: pgmap v383: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: pgmap v384: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 199 B/s rd, 0 op/s 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: pgmap v385: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:16:54.447 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.448 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:54.449 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:54 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:16:56.755 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:56 smithi144 ceph-mon[67032]: Upgrade: Updating grafana.smithi144 2024-04-12T23:16:56.755 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:56 smithi144 ceph-mon[67032]: Deploying daemon grafana.smithi144 on smithi144 2024-04-12T23:16:56.755 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:56 smithi144 ceph-mon[67032]: pgmap v386: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 124 B/s rd, 0 op/s 2024-04-12T23:16:56.755 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:56 smithi181 ceph-mon[52974]: Upgrade: Updating grafana.smithi144 2024-04-12T23:16:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:56 smithi181 ceph-mon[52974]: Deploying daemon grafana.smithi144 on smithi144 2024-04-12T23:16:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:56 smithi181 ceph-mon[52974]: pgmap v386: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 124 B/s rd, 0 op/s 2024-04-12T23:16:56.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:16:58.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:58 smithi144 ceph-mon[67032]: pgmap v387: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T23:16:58.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:58 smithi181 ceph-mon[52974]: pgmap v387: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T23:16:59.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:59.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:59.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:16:59 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:16:59.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:16:59.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:16:59 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:00.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:00 smithi181 ceph-mon[52974]: pgmap v388: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 124 B/s rd, 0 op/s 2024-04-12T23:17:00.956 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:00 smithi144 ceph-mon[67032]: pgmap v388: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 124 B/s rd, 0 op/s 2024-04-12T23:17:02.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:02 smithi144 ceph-mon[67032]: pgmap v389: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T23:17:02.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:02 smithi181 ceph-mon[52974]: pgmap v389: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 248 B/s rd, 0 op/s 2024-04-12T23:17:03.034 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:17:03.703 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:17:03.703 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (20s) 0s ago 20m 16.9M - 0.25.0 c8568f914cd2 4c7beee56a5b 2024-04-12T23:17:03.703 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (8m) 0s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:17:03.703 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (7m) 50s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:17:03.703 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (5s) 0s ago 20m 48.7M - 9.4.12 2bacad6d85d8 50711afe2de8 2024-04-12T23:17:03.703 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (117s) 0s ago 14m 36.0M - 19.0.0-2053-gb277feb2 668895944f0e 8982b4407488 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (96s) 50s ago 14m 11.5M - 19.0.0-2053-gb277feb2 668895944f0e 04579505b4e8 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (10m) 0s ago 22m 524M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (9m) 50s ago 20m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (9m) 0s ago 22m 65.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (9m) 50s ago 19m 45.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (82s) 0s ago 14m 77.6M - 5.7 668895944f0e d28a74ff9bef 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (76s) 50s ago 14m 61.7M - 5.7 668895944f0e aa8f9b410175 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (60s) 0s ago 20m 8464k - 1.5.0 0da6a335fe13 92260f489e83 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (53s) 50s ago 19m 5486k - 1.5.0 0da6a335fe13 7804af9122fe 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (7m) 0s ago 19m 427M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (6m) 0s ago 18m 383M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (5m) 0s ago 18m 394M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (4m) 0s ago 17m 421M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (4m) 50s ago 17m 371M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:17:03.704 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (3m) 50s ago 17m 326M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:17:03.705 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (3m) 50s ago 16m 309M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:17:03.705 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (2m) 50s ago 16m 297M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:17:03.705 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (37s) 0s ago 19m 51.3M - 2.43.0 a07b618ecd1d a165a1cca373 2024-04-12T23:17:04.114 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:17:04.114 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:17:04.115 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": true, 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [ 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "nfs", 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "mds", 2024-04-12T23:17:04.477 INFO:teuthology.orchestra.run.smithi144.stdout: "crash", 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: "osd", 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr", 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: "mon" 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": "18/23 daemons upgraded", 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "Currently upgrading grafana daemons", 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:17:04.478 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: pgmap v390: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:04.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:04 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3513233625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:04.915 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-12T23:17:04.915 INFO:teuthology.orchestra.run.smithi144.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-12T23:17:04.916 INFO:teuthology.orchestra.run.smithi144.stdout: daemon grafana.smithi144 on smithi144 is in error state 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: pgmap v390: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:04.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:04 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3513233625' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:05.740 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:05 smithi144 ceph-mon[67032]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:05.740 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:05 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:05.740 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:05 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:05.741 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:05 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3992566339' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:17:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:05 smithi181 ceph-mon[52974]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:05 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:05 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:05 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3992566339' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:17:06.705 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:06 smithi144 ceph-mon[67032]: pgmap v391: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:06.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:06 smithi181 ceph-mon[52974]: pgmap v391: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:08.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: pgmap v392: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:08.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.948 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-12T23:17:08.949 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-12T23:17:08.950 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.951 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.952 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:08.952 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:17:08.952 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:17:08.952 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:08.952 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:08 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: pgmap v392: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:09.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.024 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-12T23:17:09.025 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-12T23:17:09.026 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:09.027 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.028 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:08 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:09.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:09 smithi181 ceph-mon[52974]: pgmap v393: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:09 smithi181 ceph-mon[52974]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:17:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:09 smithi181 ceph-mon[52974]: Upgrade: Finalizing container_image settings 2024-04-12T23:17:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:09 smithi181 ceph-mon[52974]: Upgrade: Complete! 2024-04-12T23:17:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:09 smithi181 ceph-mon[52974]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T23:17:09.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:09 smithi181 ceph-mon[52974]: Cluster is now healthy 2024-04-12T23:17:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:09 smithi144 ceph-mon[67032]: pgmap v393: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:09 smithi144 ceph-mon[67032]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-12T23:17:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:09 smithi144 ceph-mon[67032]: Upgrade: Finalizing container_image settings 2024-04-12T23:17:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:09 smithi144 ceph-mon[67032]: Upgrade: Complete! 2024-04-12T23:17:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:09 smithi144 ceph-mon[67032]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-12T23:17:10.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:09 smithi144 ceph-mon[67032]: Cluster is now healthy 2024-04-12T23:17:11.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:11 smithi181 ceph-mon[52974]: pgmap v394: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:11.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:11.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:11.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:12.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:11 smithi144 ceph-mon[67032]: pgmap v394: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:12.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:12.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:12.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:17:13.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:13 smithi181 ceph-mon[52974]: pgmap v395: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:14.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:13 smithi144 ceph-mon[67032]: pgmap v395: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:15.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:15 smithi181 ceph-mon[52974]: pgmap v396: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:16.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:15 smithi144 ceph-mon[67032]: pgmap v396: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:17.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:17 smithi181 ceph-mon[52974]: pgmap v397: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 273 B/s rd, 0 op/s 2024-04-12T23:17:18.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:17 smithi144 ceph-mon[67032]: pgmap v397: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 273 B/s rd, 0 op/s 2024-04-12T23:17:19.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:19 smithi181 ceph-mon[52974]: pgmap v398: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:20.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:19 smithi144 ceph-mon[67032]: pgmap v398: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 182 B/s rd, 0 op/s 2024-04-12T23:17:21.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:21 smithi181 ceph-mon[52974]: pgmap v399: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:22.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:21 smithi144 ceph-mon[67032]: pgmap v399: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:23.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:23 smithi181 ceph-mon[52974]: pgmap v400: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:24.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:23 smithi144 ceph-mon[67032]: pgmap v400: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:25.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:25 smithi181 ceph-mon[52974]: pgmap v401: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:26.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:25 smithi144 ceph-mon[67032]: pgmap v401: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:26.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:27.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:27.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:27 smithi181 ceph-mon[52974]: pgmap v402: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:28.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:27 smithi144 ceph-mon[67032]: pgmap v402: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:30.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:29 smithi144 ceph-mon[67032]: pgmap v403: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:30.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:29 smithi181 ceph-mon[52974]: pgmap v403: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:32.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:31 smithi144 ceph-mon[67032]: pgmap v404: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:32.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:31 smithi181 ceph-mon[52974]: pgmap v404: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:34.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:33 smithi144 ceph-mon[67032]: pgmap v405: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:34.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:33 smithi181 ceph-mon[52974]: pgmap v405: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:36.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:35 smithi144 ceph-mon[67032]: pgmap v406: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:36.029 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T23:17:36.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:35 smithi181 ceph-mon[52974]: pgmap v406: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:37.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:36 smithi144 ceph-mon[67032]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:37.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:36 smithi181 ceph-mon[52974]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (54s) 34s ago 21m 16.9M - 0.25.0 c8568f914cd2 4c7beee56a5b 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (8m) 34s ago 21m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (8m) 84s ago 20m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (39s) 34s ago 20m 48.7M - 9.4.12 2bacad6d85d8 50711afe2de8 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (2m) 34s ago 15m 36.0M - 19.0.0-2053-gb277feb2 668895944f0e 8982b4407488 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (2m) 84s ago 15m 11.5M - 19.0.0-2053-gb277feb2 668895944f0e 04579505b4e8 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (11m) 34s ago 22m 524M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:17:38.003 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (10m) 84s ago 20m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (10m) 34s ago 22m 65.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (9m) 84s ago 20m 45.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (116s) 34s ago 15m 77.6M - 5.7 668895944f0e d28a74ff9bef 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (110s) 84s ago 15m 61.7M - 5.7 668895944f0e aa8f9b410175 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (94s) 34s ago 20m 8464k - 1.5.0 0da6a335fe13 92260f489e83 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (88s) 84s ago 20m 5486k - 1.5.0 0da6a335fe13 7804af9122fe 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (7m) 34s ago 19m 427M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (7m) 34s ago 19m 383M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (6m) 34s ago 18m 394M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (5m) 34s ago 18m 421M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (4m) 84s ago 18m 371M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (4m) 84s ago 17m 326M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (3m) 84s ago 17m 309M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (2m) 84s ago 16m 297M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:17:38.004 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (71s) 34s ago 20m 51.3M - 2.43.0 a07b618ecd1d a165a1cca373 2024-04-12T23:17:38.005 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:37 smithi144 ceph-mon[67032]: pgmap v407: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:38.198 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:37 smithi181 ceph-mon[52974]: pgmap v407: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:38.601 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions' 2024-04-12T23:17:39.976 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:39 smithi144 ceph-mon[67032]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:39.976 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:39 smithi144 ceph-mon[67032]: pgmap v408: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:40.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:39 smithi181 ceph-mon[52974]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:17:40.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:39 smithi181 ceph-mon[52974]: pgmap v408: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:40.581 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:17:40.582 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:17:40.582 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:17:40.582 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:17:40.582 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T23:17:40.582 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:17:40.582 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:17:40.953 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:40 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/1443086558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:41.098 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:40 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/1443086558' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:17:41.241 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-12T23:17:41.838 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:41 smithi144 ceph-mon[67032]: pgmap v409: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:41.839 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:42.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:41 smithi181 ceph-mon[52974]: pgmap v409: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:42.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:42.945 INFO:teuthology.orchestra.run.smithi144.stdout:wait for servicemap items w/ changing names to refresh 2024-04-12T23:17:43.591 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'sleep 60' 2024-04-12T23:17:43.892 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:43 smithi144 ceph-mon[67032]: pgmap v410: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:44.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:43 smithi181 ceph-mon[52974]: pgmap v410: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:46.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:45 smithi144 ceph-mon[67032]: pgmap v411: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:46.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:45 smithi181 ceph-mon[52974]: pgmap v411: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:48.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:47 smithi144 ceph-mon[67032]: pgmap v412: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:48.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:47 smithi181 ceph-mon[52974]: pgmap v412: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:50.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:49 smithi144 ceph-mon[67032]: pgmap v413: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:50.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:49 smithi181 ceph-mon[52974]: pgmap v413: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:52.022 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:51 smithi144 ceph-mon[67032]: pgmap v414: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:52.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:51 smithi181 ceph-mon[52974]: pgmap v414: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:54.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:53 smithi144 ceph-mon[67032]: pgmap v415: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:54.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:53 smithi181 ceph-mon[52974]: pgmap v415: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:56.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:55 smithi144 ceph-mon[67032]: pgmap v416: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:56.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:55 smithi181 ceph-mon[52974]: pgmap v416: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:57.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:57.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:17:58.023 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:57 smithi144 ceph-mon[67032]: pgmap v417: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:58.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:57 smithi181 ceph-mon[52974]: pgmap v417: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:17:59.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:17:58 smithi181 ceph-mon[52974]: pgmap v418: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:17:59.273 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:17:58 smithi144 ceph-mon[67032]: pgmap v418: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:01.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:01 smithi181 ceph-mon[52974]: pgmap v419: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:01.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:01 smithi144 ceph-mon[67032]: pgmap v419: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:03.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:03 smithi181 ceph-mon[52974]: pgmap v420: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:03.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:03 smithi144 ceph-mon[67032]: pgmap v420: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:05.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:05 smithi181 ceph-mon[52974]: pgmap v421: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:05.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:05 smithi144 ceph-mon[67032]: pgmap v421: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:07.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:07 smithi181 ceph-mon[52974]: pgmap v422: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:07.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:07 smithi144 ceph-mon[67032]: pgmap v422: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:09.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:09 smithi181 ceph-mon[52974]: pgmap v423: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:09.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:09 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:18:09.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:09 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:09.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:09 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:09.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:09 smithi144 ceph-mon[67032]: pgmap v423: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:09.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:09 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:18:09.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:09 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:09.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:09 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:10.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:18:10.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:18:10.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:10.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:10.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:18:10.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:18:10.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:10.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:18:11.598 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:11 smithi181 ceph-mon[52974]: pgmap v424: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:11.598 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:11.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:11 smithi144 ceph-mon[67032]: pgmap v424: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:11.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:13.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:13 smithi181 ceph-mon[52974]: pgmap v425: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:13.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:13 smithi144 ceph-mon[67032]: pgmap v425: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:15.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:15 smithi181 ceph-mon[52974]: pgmap v426: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:15 smithi144 ceph-mon[67032]: pgmap v426: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:17.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:17 smithi181 ceph-mon[52974]: pgmap v427: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:17.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:17 smithi144 ceph-mon[67032]: pgmap v427: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:19.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:19 smithi181 ceph-mon[52974]: pgmap v428: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:19.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:19 smithi144 ceph-mon[67032]: pgmap v428: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:21.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:21 smithi181 ceph-mon[52974]: pgmap v429: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:21.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:21 smithi144 ceph-mon[67032]: pgmap v429: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:23.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:23 smithi181 ceph-mon[52974]: pgmap v430: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:23.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:23 smithi144 ceph-mon[67032]: pgmap v430: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:25.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:25 smithi181 ceph-mon[52974]: pgmap v431: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:25.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:25 smithi144 ceph-mon[67032]: pgmap v431: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:26.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:26 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:26.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:26 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:27.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:27 smithi181 ceph-mon[52974]: pgmap v432: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:27.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:27 smithi144 ceph-mon[67032]: pgmap v432: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:29.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:29 smithi181 ceph-mon[52974]: pgmap v433: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:29.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:29 smithi144 ceph-mon[67032]: pgmap v433: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:31.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:31 smithi181 ceph-mon[52974]: pgmap v434: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:31.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:31 smithi144 ceph-mon[67032]: pgmap v434: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:33.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:33 smithi181 ceph-mon[52974]: pgmap v435: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:33.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:33 smithi144 ceph-mon[67032]: pgmap v435: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:35.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:35 smithi181 ceph-mon[52974]: pgmap v436: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:35.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:35 smithi144 ceph-mon[67032]: pgmap v436: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:37.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:37 smithi181 ceph-mon[52974]: pgmap v437: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:37.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:37 smithi144 ceph-mon[67032]: pgmap v437: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:39.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:39 smithi181 ceph-mon[52974]: pgmap v438: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:39.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:39 smithi144 ceph-mon[67032]: pgmap v438: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:41.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:41 smithi181 ceph-mon[52974]: pgmap v439: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:41.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:41 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:41.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:41 smithi144 ceph-mon[67032]: pgmap v439: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:41.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:41 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:43.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:43 smithi181 ceph-mon[52974]: pgmap v440: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:43.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:43 smithi144 ceph-mon[67032]: pgmap v440: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:45.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:45 smithi144 ceph-mon[67032]: pgmap v441: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:45.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:45 smithi181 ceph-mon[52974]: pgmap v441: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:45.848 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch ps' 2024-04-12T23:18:47.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:47 smithi181 ceph-mon[52974]: pgmap v442: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:47.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:47 smithi144 ceph-mon[67032]: pgmap v442: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:47.833 INFO:teuthology.orchestra.run.smithi144.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-12T23:18:47.833 INFO:teuthology.orchestra.run.smithi144.stdout:alertmanager.smithi144 smithi144 *:9093,9094 running (2m) 104s ago 22m 16.9M - 0.25.0 c8568f914cd2 4c7beee56a5b 2024-04-12T23:18:47.833 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi144 smithi144 running (9m) 104s ago 22m 6665k - 19.0.0-2053-gb277feb2 668895944f0e 7c562b7416d1 2024-04-12T23:18:47.833 INFO:teuthology.orchestra.run.smithi144.stdout:crash.smithi181 smithi181 running (9m) 2m ago 21m 6665k - 19.0.0-2053-gb277feb2 668895944f0e d5a8e688e903 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:grafana.smithi144 smithi144 *:3000 running (109s) 104s ago 22m 48.7M - 9.4.12 2bacad6d85d8 50711afe2de8 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi144.alilcy smithi144 running (3m) 104s ago 16m 36.0M - 19.0.0-2053-gb277feb2 668895944f0e 8982b4407488 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:mds.foofs.smithi181.gciyns smithi181 running (3m) 2m ago 16m 11.5M - 19.0.0-2053-gb277feb2 668895944f0e 04579505b4e8 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi144.rlueky smithi144 *:8443,9283,8765 running (12m) 104s ago 23m 524M - 19.0.0-2053-gb277feb2 668895944f0e 22d900813d08 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:mgr.smithi181.whsrmz smithi181 *:8443,9283,8765 running (11m) 2m ago 21m 453M - 19.0.0-2053-gb277feb2 668895944f0e e6a8517f5653 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi144 smithi144 running (11m) 104s ago 23m 65.0M 2048M 19.0.0-2053-gb277feb2 668895944f0e 70ffa37b2714 2024-04-12T23:18:47.834 INFO:teuthology.orchestra.run.smithi144.stdout:mon.smithi181 smithi181 running (10m) 2m ago 21m 45.5M 2048M 19.0.0-2053-gb277feb2 668895944f0e f55287e68062 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.0.0.smithi144.mjwmkt smithi144 *:2049 running (3m) 104s ago 16m 77.6M - 5.7 668895944f0e d28a74ff9bef 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:nfs.foo.1.0.smithi181.dgxqfb smithi181 *:2049 running (3m) 2m ago 16m 61.7M - 5.7 668895944f0e aa8f9b410175 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi144 smithi144 *:9100 running (2m) 104s ago 22m 8464k - 1.5.0 0da6a335fe13 92260f489e83 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:node-exporter.smithi181 smithi181 *:9100 running (2m) 2m ago 21m 5486k - 1.5.0 0da6a335fe13 7804af9122fe 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:osd.0 smithi144 running (9m) 104s ago 20m 427M 1936M 19.0.0-2053-gb277feb2 668895944f0e 30a88374fddd 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:osd.1 smithi144 running (8m) 104s ago 20m 383M 1936M 19.0.0-2053-gb277feb2 668895944f0e 89e5c1a369f7 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:osd.2 smithi144 running (7m) 104s ago 20m 394M 1936M 19.0.0-2053-gb277feb2 668895944f0e 8af22348abd2 2024-04-12T23:18:47.835 INFO:teuthology.orchestra.run.smithi144.stdout:osd.3 smithi144 running (6m) 104s ago 19m 421M 1936M 19.0.0-2053-gb277feb2 668895944f0e f7bec85ff725 2024-04-12T23:18:47.836 INFO:teuthology.orchestra.run.smithi144.stdout:osd.4 smithi181 running (6m) 2m ago 19m 371M 2704M 19.0.0-2053-gb277feb2 668895944f0e d5b22101fa21 2024-04-12T23:18:47.836 INFO:teuthology.orchestra.run.smithi144.stdout:osd.5 smithi181 running (5m) 2m ago 18m 326M 2704M 19.0.0-2053-gb277feb2 668895944f0e d4863d7c5c03 2024-04-12T23:18:47.836 INFO:teuthology.orchestra.run.smithi144.stdout:osd.6 smithi181 running (4m) 2m ago 18m 309M 2704M 19.0.0-2053-gb277feb2 668895944f0e 6c1178b4b3f3 2024-04-12T23:18:47.836 INFO:teuthology.orchestra.run.smithi144.stdout:osd.7 smithi181 running (4m) 2m ago 18m 297M 2704M 19.0.0-2053-gb277feb2 668895944f0e 54cb91ee3688 2024-04-12T23:18:47.836 INFO:teuthology.orchestra.run.smithi144.stdout:prometheus.smithi144 smithi144 *:9095 running (2m) 104s ago 21m 51.3M - 2.43.0 a07b618ecd1d a165a1cca373 2024-04-12T23:18:48.526 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph orch upgrade status' 2024-04-12T23:18:49.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:49 smithi144 ceph-mon[67032]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:18:49.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:49 smithi144 ceph-mon[67032]: pgmap v443: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:49.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:49 smithi181 ceph-mon[52974]: from='client.34640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:18:49.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:49 smithi181 ceph-mon[52974]: pgmap v443: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:50.539 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "target_image": null, 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "in_progress": false, 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "which": "", 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "services_complete": [], 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "progress": null, 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "message": "", 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout: "is_paused": false 2024-04-12T23:18:50.540 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:18:51.154 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph health detail' 2024-04-12T23:18:51.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:51 smithi181 ceph-mon[52974]: pgmap v444: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:51.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:51 smithi181 ceph-mon[52974]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:18:51.716 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:51 smithi144 ceph-mon[67032]: pgmap v444: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:51.717 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:51 smithi144 ceph-mon[67032]: from='client.34644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:18:53.237 INFO:teuthology.orchestra.run.smithi144.stdout:HEALTH_OK 2024-04-12T23:18:53.522 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:53 smithi144 ceph-mon[67032]: pgmap v445: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:53.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:53 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/4022168817' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:18:53.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:53 smithi181 ceph-mon[52974]: pgmap v445: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:53.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:53 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/4022168817' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-12T23:18:53.951 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions' 2024-04-12T23:18:55.523 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:55 smithi144 ceph-mon[67032]: pgmap v446: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:55.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:55 smithi181 ceph-mon[52974]: pgmap v446: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:55.934 INFO:teuthology.orchestra.run.smithi144.stdout:{ 2024-04-12T23:18:55.934 INFO:teuthology.orchestra.run.smithi144.stdout: "mon": { 2024-04-12T23:18:55.934 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:18:55.934 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:18:55.935 INFO:teuthology.orchestra.run.smithi144.stdout: "mgr": { 2024-04-12T23:18:55.935 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:18:55.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:18:55.935 INFO:teuthology.orchestra.run.smithi144.stdout: "osd": { 2024-04-12T23:18:55.935 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 8 2024-04-12T23:18:55.935 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout: "mds": { 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 2 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout: "overall": { 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)": 14 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:18:55.936 INFO:teuthology.orchestra.run.smithi144.stdout:} 2024-04-12T23:18:56.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:56 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2311767133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:18:56.535 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:56 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:56.558 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-12T23:18:56.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:56 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2311767133' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:18:56.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:56 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:18:57.522 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:57 smithi144 ceph-mon[67032]: pgmap v447: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:57.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:57 smithi181 ceph-mon[52974]: pgmap v447: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:18:58.673 INFO:teuthology.orchestra.run.smithi144.stdout:true 2024-04-12T23:18:59.376 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -e sha1=b277feb2c7beb4597d8ecc39d58d5c36b81f613a -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-12T23:18:59.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:59 smithi144 ceph-mon[67032]: pgmap v448: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:59.684 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:18:59 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/173422241' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:18:59.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:59 smithi181 ceph-mon[52974]: pgmap v448: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:18:59.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:18:59 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/173422241' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:19:01.432 INFO:teuthology.orchestra.run.smithi144.stdout: "ceph version 19.0.0-2053-gb277feb2 (b277feb2c7beb4597d8ecc39d58d5c36b81f613a) squid (dev)" 2024-04-12T23:19:01.722 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:01 smithi144 ceph-mon[67032]: pgmap v449: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:01.723 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:01 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/3291177244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:19:01.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:01 smithi181 ceph-mon[52974]: pgmap v449: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:01.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:01 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/3291177244' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-12T23:19:02.038 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-04-12T23:19:02.039 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-04-12T23:19:02.039 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- ceph orch ls -f json 2024-04-12T23:19:03.565 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:03 smithi144 ceph-mon[67032]: pgmap v450: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:19:03.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:03 smithi181 ceph-mon[52974]: pgmap v450: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:19:03.981 INFO:teuthology.orchestra.run.smithi144.stdout: 2024-04-12T23:19:03.981 INFO:teuthology.orchestra.run.smithi144.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-04-12T22:55:48.243697Z", "last_refresh": "2024-04-12T23:17:03.660133Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-04-12T22:55:42.567016Z", "last_refresh": "2024-04-12T23:16:13.445719Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-04-12T22:55:45.775050Z", "last_refresh": "2024-04-12T23:17:03.660276Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-04-12T23:02:03.365164Z", "last_refresh": "2024-04-12T23:16:13.446348Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-04-12T22:55:40.927783Z", "last_refresh": "2024-04-12T23:16:13.445820Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi144:172.21.15.144=smithi144", "smithi181:172.21.15.181=smithi181"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-04-12T22:56:30.064757Z", "last_refresh": "2024-04-12T23:16:13.445900Z", "running": 2, "size": 2}}, {"events": ["2024-04-12T23:18:09.796826Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-04-12T23:02:22.838287Z", "last_refresh": "2024-04-12T23:16:13.446423Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-04-12T22:55:46.969530Z", "last_refresh": "2024-04-12T23:16:13.445977Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec", "last_refresh": "2024-04-12T23:16:13.446051Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-04-12T22:55:44.565108Z", "last_refresh": "2024-04-12T23:17:03.660419Z", "ports": [9095], "running": 1, "size": 1}}] 2024-04-12T23:19:04.747 INFO:tasks.cephadm:nfs.foo has 2/2 2024-04-12T23:19:04.748 DEBUG:teuthology.parallel:result is None 2024-04-12T23:19:04.748 INFO:teuthology.run_tasks:Running task vip.exec... 2024-04-12T23:19:04.760 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:19:04.761 DEBUG:teuthology.orchestra.run.smithi144:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-04-12T23:19:04.790 INFO:teuthology.orchestra.run.smithi144.stderr:+ umount /mnt/foo 2024-04-12T23:19:04.801 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T23:19:04.811 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:19:04.812 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-04-12T23:19:05.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:05 smithi144 ceph-mon[67032]: from='client.34664 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:19:05.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:05 smithi144 ceph-mon[67032]: pgmap v451: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:05.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:05 smithi181 ceph-mon[52974]: from='client.34664 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-12T23:19:05.947 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:05 smithi181 ceph-mon[52974]: pgmap v451: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:06.737 INFO:teuthology.orchestra.run.smithi144.stdout: "foo" 2024-04-12T23:19:07.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-04-12T23:19:07.667 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:07 smithi144 ceph-mon[67032]: pgmap v452: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:19:07.667 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:07 smithi144 ceph-mon[67032]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:19:07.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:07 smithi181 ceph-mon[52974]: pgmap v452: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:19:07.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:07 smithi181 ceph-mon[52974]: from='client.34668 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:19:09.280 INFO:teuthology.orchestra.run.smithi144.stdout:[ 2024-04-12T23:19:09.280 INFO:teuthology.orchestra.run.smithi144.stdout: { 2024-04-12T23:19:09.280 INFO:teuthology.orchestra.run.smithi144.stdout: "access_type": "RW", 2024-04-12T23:19:09.280 INFO:teuthology.orchestra.run.smithi144.stdout: "clients": [], 2024-04-12T23:19:09.280 INFO:teuthology.orchestra.run.smithi144.stdout: "cluster_id": "foo", 2024-04-12T23:19:09.280 INFO:teuthology.orchestra.run.smithi144.stdout: "export_id": 1, 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "fsal": { 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "fs_name": "foofs", 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "name": "CEPH", 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "user_id": "nfs.foo.1" 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: }, 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "path": "/", 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "protocols": [ 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: 4 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: ], 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "pseudo": "/fake", 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "security_label": true, 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "squash": "none", 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "transports": [ 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: "TCP" 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: ] 2024-04-12T23:19:09.281 INFO:teuthology.orchestra.run.smithi144.stdout: } 2024-04-12T23:19:09.282 INFO:teuthology.orchestra.run.smithi144.stdout:] 2024-04-12T23:19:09.724 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:09 smithi144 ceph-mon[67032]: pgmap v453: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:09.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:09 smithi181 ceph-mon[52974]: pgmap v453: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:10.033 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-04-12T23:19:10.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:10 smithi144 ceph-mon[67032]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:19:10.678 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:10 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:19:10.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:10 smithi181 ceph-mon[52974]: from='client.34672 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-12T23:19:10.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:10 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-12T23:19:11.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:11 smithi144 ceph-mon[67032]: pgmap v454: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:11.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:19:11.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:19:11.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:19:11.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:19:11.567 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:11 smithi144 ceph-mon[67032]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:19:11.787 INFO:teuthology.orchestra.run.smithi144.stdout:foo rec-0000000000000004:nfs.foo.1 2024-04-12T23:19:11.788 INFO:teuthology.orchestra.run.smithi144.stdout:foo grace 2024-04-12T23:19:11.788 INFO:teuthology.orchestra.run.smithi144.stdout:foo rec-0000000000000004:nfs.foo.0 2024-04-12T23:19:11.788 INFO:teuthology.orchestra.run.smithi144.stdout:foo conf-nfs.foo 2024-04-12T23:19:11.788 INFO:teuthology.orchestra.run.smithi144.stdout:foo export-1 2024-04-12T23:19:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:11 smithi181 ceph-mon[52974]: pgmap v454: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-12T23:19:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-12T23:19:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-12T23:19:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:19:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' 2024-04-12T23:19:11.848 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:11 smithi181 ceph-mon[52974]: from='mgr.34104 172.21.15.144:0/2597479009' entity='mgr.smithi144.rlueky' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-12T23:19:12.391 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-12T23:19:12.400 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:19:12.400 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 -- bash -c 'set -ex 2024-04-12T23:19:12.401 DEBUG:teuthology.orchestra.run.smithi144:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-04-12T23:19:12.401 DEBUG:teuthology.orchestra.run.smithi144:> ' 2024-04-12T23:19:12.937 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:12 smithi144 ceph-mon[67032]: mgrmap e43: smithi144.rlueky(active, since 11m), standbys: smithi181.whsrmz 2024-04-12T23:19:12.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:12 smithi181 ceph-mon[52974]: mgrmap e43: smithi144.rlueky(active, since 11m), standbys: smithi181.whsrmz 2024-04-12T23:19:13.682 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:13 smithi144 ceph-mon[67032]: pgmap v455: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:19:13.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:13 smithi181 ceph-mon[52974]: pgmap v455: 97 pgs: 97 active+clean; 228 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-04-12T23:19:14.144 INFO:teuthology.orchestra.run.smithi144.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-04-12T23:19:14.389 INFO:teuthology.orchestra.run.smithi144.stderr:+ [[ 6 -gt 2 ]] 2024-04-12T23:19:14.946 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:14 smithi181 ceph-mon[52974]: from='client.? 172.21.15.144:0/2993649466' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-12T23:19:15.006 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:14 smithi144 ceph-mon[67032]: from='client.? 172.21.15.144:0/2993649466' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-04-12T23:19:15.030 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-12T23:19:15.040 INFO:tasks.cephadm:Teardown begin 2024-04-12T23:19:15.040 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T23:19:15.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T23:19:15.105 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-12T23:19:15.105 DEBUG:teuthology.orchestra.run.smithi144:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T23:19:15.126 DEBUG:teuthology.orchestra.run.smithi181:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-12T23:19:15.161 INFO:tasks.cephadm:Stopping all daemons... 2024-04-12T23:19:15.162 INFO:tasks.cephadm.mon.smithi144:Stopping mon.smithi144... 2024-04-12T23:19:15.162 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144 2024-04-12T23:19:15.272 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:15 smithi144 systemd[1]: Stopping Ceph mon.smithi144 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:15.772 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[67028]: 2024-04-12T23:19:15.488+0000 7ff52a4b3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi144 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:15 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144[67028]: 2024-04-12T23:19:15.488+0000 7ff52a4b3700 -1 mon.smithi144@0(leader) e3 *** Got Signal Terminated *** 2024-04-12T23:19:15.773 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:15 smithi144 podman[112365]: 2024-04-12 23:19:15.520467245 +0000 UTC m=+0.155907961 container died 70ffa37b271436f8a5f89a8139044746da10c894e0a23fd6e6f070ab5358a1a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:16.186 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:15 smithi144 podman[112365]: 2024-04-12 23:19:15.898436436 +0000 UTC m=+0.533877147 container cleanup 70ffa37b271436f8a5f89a8139044746da10c894e0a23fd6e6f070ab5358a1a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.build-date=20240409) 2024-04-12T23:19:16.186 INFO:journalctl@ceph.mon.smithi144.smithi144.stdout:Apr 12 23:19:15 smithi144 bash[112365]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi144 2024-04-12T23:19:16.359 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi144.service' 2024-04-12T23:19:16.412 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:16.412 INFO:tasks.cephadm.mon.smithi144:Stopped mon.smithi144 2024-04-12T23:19:16.412 INFO:tasks.cephadm.mon.smithi181:Stopping mon.smithi181... 2024-04-12T23:19:16.412 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi181 2024-04-12T23:19:16.697 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:16 smithi181 systemd[1]: Stopping Ceph mon.smithi181 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:17.196 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181[52970]: 2024-04-12T23:19:16.742+0000 7f251c5eb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi181 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:17.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181[52970]: 2024-04-12T23:19:16.742+0000 7f251c5eb700 -1 mon.smithi181@1(peon) e3 *** Got Signal Terminated *** 2024-04-12T23:19:17.197 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:16 smithi181 podman[78763]: 2024-04-12 23:19:16.957841894 +0000 UTC m=+0.335406104 container died f55287e68062b894244fde061ece6000cbf4a5ad832d9eb660820e9511b1dca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:17.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:17 smithi181 podman[78763]: 2024-04-12 23:19:17.418067868 +0000 UTC m=+0.795632148 container cleanup f55287e68062b894244fde061ece6000cbf4a5ad832d9eb660820e9511b1dca1 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-12T23:19:17.696 INFO:journalctl@ceph.mon.smithi181.smithi181.stdout:Apr 12 23:19:17 smithi181 bash[78763]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-mon-smithi181 2024-04-12T23:19:17.836 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@mon.smithi181.service' 2024-04-12T23:19:17.911 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:17.911 INFO:tasks.cephadm.mon.smithi181:Stopped mon.smithi181 2024-04-12T23:19:17.911 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-12T23:19:17.911 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.0 2024-04-12T23:19:18.272 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:17 smithi144 systemd[1]: Stopping Ceph osd.0 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:18.772 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:19:18.284+0000 7fe333290700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:18.773 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:19:18.284+0000 7fe333290700 -1 osd.0 159 *** Got signal Terminated *** 2024-04-12T23:19:18.773 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:18 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0[73539]: 2024-04-12T23:19:18.284+0000 7fe333290700 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:19:23.932 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:23 smithi144 podman[112528]: 2024-04-12 23:19:23.693472506 +0000 UTC m=+5.566207579 container died 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409) 2024-04-12T23:19:24.273 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112528]: 2024-04-12 23:19:24.060510615 +0000 UTC m=+5.933245684 container cleanup 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS) 2024-04-12T23:19:24.273 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 bash[112528]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0 2024-04-12T23:19:24.641 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112545]: 2024-04-12 23:19:24.345405487 +0000 UTC m=+0.651389346 container remove 30a88374fddd876a7d3c2ca1d2063658452f4aa0394e8d5fe2c9c8f9c68fdab9 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb) 2024-04-12T23:19:24.919 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112639]: 2024-04-12 23:19:24.654809967 +0000 UTC m=+0.111470940 container create 57b767a0d7aff5413c46bae456815a7e77f3a7d13b485daf2f4dd0dc5f849514 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True) 2024-04-12T23:19:24.919 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112639]: 2024-04-12 23:19:24.561724383 +0000 UTC m=+0.018385373 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:19:24.919 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112639]: 2024-04-12 23:19:24.858076449 +0000 UTC m=+0.314737412 container init 57b767a0d7aff5413c46bae456815a7e77f3a7d13b485daf2f4dd0dc5f849514 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, ceph=True) 2024-04-12T23:19:24.919 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112639]: 2024-04-12 23:19:24.86141323 +0000 UTC m=+0.318074202 container start 57b767a0d7aff5413c46bae456815a7e77f3a7d13b485daf2f4dd0dc5f849514 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, RELEASE=squid-b277feb) 2024-04-12T23:19:24.919 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:24 smithi144 podman[112639]: 2024-04-12 23:19:24.932318405 +0000 UTC m=+0.388979371 container attach 57b767a0d7aff5413c46bae456815a7e77f3a7d13b485daf2f4dd0dc5f849514 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:25.273 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:25 smithi144 podman[112639]: 2024-04-12 23:19:25.156171437 +0000 UTC m=+0.612832410 container died 57b767a0d7aff5413c46bae456815a7e77f3a7d13b485daf2f4dd0dc5f849514 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-b277feb) 2024-04-12T23:19:25.724 INFO:journalctl@ceph.osd.0.smithi144.stdout:Apr 12 23:19:25 smithi144 podman[112666]: 2024-04-12 23:19:25.735928724 +0000 UTC m=+0.571030912 container remove 57b767a0d7aff5413c46bae456815a7e77f3a7d13b485daf2f4dd0dc5f849514 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-12T23:19:25.741 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.0.service' 2024-04-12T23:19:25.782 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:25.783 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-12T23:19:25.783 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-12T23:19:25.783 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.1 2024-04-12T23:19:26.022 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:25 smithi144 systemd[1]: Stopping Ceph osd.1 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:26.523 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:26 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:19:26.121+0000 7f6747e9f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:26.523 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:26 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:19:26.121+0000 7f6747e9f700 -1 osd.1 159 *** Got signal Terminated *** 2024-04-12T23:19:26.523 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:26 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1[76089]: 2024-04-12T23:19:26.121+0000 7f6747e9f700 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:19:31.925 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:31 smithi144 podman[112746]: 2024-04-12 23:19:31.635639305 +0000 UTC m=+5.612237892 container died 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-12T23:19:32.243 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112746]: 2024-04-12 23:19:32.038554143 +0000 UTC m=+6.015152720 container cleanup 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb) 2024-04-12T23:19:32.243 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 bash[112746]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1 2024-04-12T23:19:32.522 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112764]: 2024-04-12 23:19:32.256699805 +0000 UTC m=+0.616304964 container remove 89e5c1a369f73bfdc21555957ddf939c527242509960db4449d5778a8f9df248 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-12T23:19:32.805 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112858]: 2024-04-12 23:19:32.464399653 +0000 UTC m=+0.018356949 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:19:32.805 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112858]: 2024-04-12 23:19:32.607735511 +0000 UTC m=+0.161692842 container create 9b6df0e434aaa3de907c9d17e6eac18e62499fe9fac12a19501f9d8548ffa2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:19:32.805 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112858]: 2024-04-12 23:19:32.818551515 +0000 UTC m=+0.372508802 container init 9b6df0e434aaa3de907c9d17e6eac18e62499fe9fac12a19501f9d8548ffa2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, ceph=True) 2024-04-12T23:19:33.098 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112858]: 2024-04-12 23:19:32.82197179 +0000 UTC m=+0.375929087 container start 9b6df0e434aaa3de907c9d17e6eac18e62499fe9fac12a19501f9d8548ffa2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-b277feb, ceph=True) 2024-04-12T23:19:33.098 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:32 smithi144 podman[112858]: 2024-04-12 23:19:32.89337945 +0000 UTC m=+0.447336737 container attach 9b6df0e434aaa3de907c9d17e6eac18e62499fe9fac12a19501f9d8548ffa2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:33.383 INFO:journalctl@ceph.osd.1.smithi144.stdout:Apr 12 23:19:33 smithi144 podman[112858]: 2024-04-12 23:19:33.112495672 +0000 UTC m=+0.666452961 container died 9b6df0e434aaa3de907c9d17e6eac18e62499fe9fac12a19501f9d8548ffa2f3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-1-deactivate, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:19:33.701 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.1.service' 2024-04-12T23:19:33.743 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:33.743 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-12T23:19:33.743 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-12T23:19:33.743 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2 2024-04-12T23:19:34.022 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:33 smithi144 systemd[1]: Stopping Ceph osd.2 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:34.522 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:34 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:19:34.125+0000 7f39a28c4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:34.523 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:34 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:19:34.125+0000 7f39a28c4700 -1 osd.2 159 *** Got signal Terminated *** 2024-04-12T23:19:34.523 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:34 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2[78772]: 2024-04-12T23:19:34.125+0000 7f39a28c4700 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:19:39.971 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:39 smithi144 podman[112964]: 2024-04-12 23:19:39.706359322 +0000 UTC m=+5.724471527 container died 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b277feb, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:40.273 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 podman[112964]: 2024-04-12 23:19:40.058199116 +0000 UTC m=+6.076311303 container cleanup 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, org.label-schema.build-date=20240409, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-12T23:19:40.273 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 bash[112964]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2 2024-04-12T23:19:40.664 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 podman[113016]: 2024-04-12 23:19:40.35147408 +0000 UTC m=+0.639255350 container remove 8af22348abd27f29443df5142af35d0195dad96bc15d57ac54cec991eb8aace3 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240409) 2024-04-12T23:19:41.000 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 podman[113109]: 2024-04-12 23:19:40.580966113 +0000 UTC m=+0.018570417 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:19:41.000 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 podman[113109]: 2024-04-12 23:19:40.694389054 +0000 UTC m=+0.131993397 container create 50fb15c7b62fd0ec26f7b724b2dcf8ef4ab2256ee925b01c3d7e1d43d968b546 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0) 2024-04-12T23:19:41.001 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 podman[113109]: 2024-04-12 23:19:40.91336065 +0000 UTC m=+0.350964950 container init 50fb15c7b62fd0ec26f7b724b2dcf8ef4ab2256ee925b01c3d7e1d43d968b546 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-12T23:19:41.001 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:40 smithi144 podman[113109]: 2024-04-12 23:19:40.916401225 +0000 UTC m=+0.354005525 container start 50fb15c7b62fd0ec26f7b724b2dcf8ef4ab2256ee925b01c3d7e1d43d968b546 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-12T23:19:41.001 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:41 smithi144 podman[113109]: 2024-04-12 23:19:41.013425047 +0000 UTC m=+0.451029371 container attach 50fb15c7b62fd0ec26f7b724b2dcf8ef4ab2256ee925b01c3d7e1d43d968b546 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-12T23:19:41.272 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:41 smithi144 podman[113109]: 2024-04-12 23:19:41.215265315 +0000 UTC m=+0.652869619 container died 50fb15c7b62fd0ec26f7b724b2dcf8ef4ab2256ee925b01c3d7e1d43d968b546 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-04-12T23:19:41.730 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2.service' 2024-04-12T23:19:41.755 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:41 smithi144 podman[113135]: 2024-04-12 23:19:41.725256153 +0000 UTC m=+0.504974250 container remove 50fb15c7b62fd0ec26f7b724b2dcf8ef4ab2256ee925b01c3d7e1d43d968b546 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux ) 2024-04-12T23:19:41.756 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:41 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-12T23:19:41.756 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:41 smithi144 systemd[1]: Stopped Ceph osd.2 for 9965523c-f91f-11ee-bc8d-c7b262605968. 2024-04-12T23:19:41.756 INFO:journalctl@ceph.osd.2.smithi144.stdout:Apr 12 23:19:41 smithi144 systemd[1]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.2.service: Consumed 1min 36.657s CPU time. 2024-04-12T23:19:41.772 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:41.772 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-12T23:19:41.772 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-12T23:19:41.772 DEBUG:teuthology.orchestra.run.smithi144:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.3 2024-04-12T23:19:42.023 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:41 smithi144 systemd[1]: Stopping Ceph osd.3 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:42.522 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:42 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:19:42.103+0000 7f369c0a5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:42.522 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:42 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:19:42.103+0000 7f369c0a5700 -1 osd.3 159 *** Got signal Terminated *** 2024-04-12T23:19:42.523 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:42 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:19:42.103+0000 7f369c0a5700 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:19:44.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:44 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:44.683+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:45.272 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:44 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:19:44.796+0000 7f3698699700 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.795639+0000 front 2024-04-12T23:19:19.795787+0000 (oldest deadline 2024-04-12T23:19:44.495738+0000) 2024-04-12T23:19:46.094 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:45 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:45.727+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:46.094 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:45 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:45.745+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:46.272 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:45 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:19:45.833+0000 7f3698699700 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.795639+0000 front 2024-04-12T23:19:19.795787+0000 (oldest deadline 2024-04-12T23:19:44.495738+0000) 2024-04-12T23:19:46.992 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:46.727+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:46.992 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:46 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:46.748+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:47.273 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:46 smithi144 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3[81321]: 2024-04-12T23:19:46.834+0000 7f3698699700 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.795639+0000 front 2024-04-12T23:19:19.795787+0000 (oldest deadline 2024-04-12T23:19:44.495738+0000) 2024-04-12T23:19:47.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:47 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:47.215+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:47.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:47 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:47.005+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:47.817 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:47 smithi144 podman[113216]: 2024-04-12 23:19:47.548404361 +0000 UTC m=+5.544693976 container died f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T23:19:47.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:47 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:47.679+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:47.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:47 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:47.736+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:48.150 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:47 smithi144 podman[113216]: 2024-04-12 23:19:47.904388957 +0000 UTC m=+5.900678579 container cleanup f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-b277feb) 2024-04-12T23:19:48.151 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:47 smithi144 bash[113216]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3 2024-04-12T23:19:48.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:48 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:48.000+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:48.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:48 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:48.177+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:48.451 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113234]: 2024-04-12 23:19:48.164030562 +0000 UTC m=+0.612317274 container remove f7bec85ff72560c2f9e23f6495d49def00f9d39bf46450d698a5387c5ca12689 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:48.729 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113328]: 2024-04-12 23:19:48.464875847 +0000 UTC m=+0.105450535 container create 905d6aaf84a237cc8b01f0a47aade2486986a2f4c496c6b8908303444fffa0a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:48.729 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113328]: 2024-04-12 23:19:48.377231944 +0000 UTC m=+0.017806646 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:19:48.729 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113328]: 2024-04-12 23:19:48.667622496 +0000 UTC m=+0.308197183 container init 905d6aaf84a237cc8b01f0a47aade2486986a2f4c496c6b8908303444fffa0a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-12T23:19:48.729 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113328]: 2024-04-12 23:19:48.67071218 +0000 UTC m=+0.311286869 container start 905d6aaf84a237cc8b01f0a47aade2486986a2f4c496c6b8908303444fffa0a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T23:19:48.729 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113328]: 2024-04-12 23:19:48.74231039 +0000 UTC m=+0.382885097 container attach 905d6aaf84a237cc8b01f0a47aade2486986a2f4c496c6b8908303444fffa0a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_CLEAN=True) 2024-04-12T23:19:48.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:48 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:48.698+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:48.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:48 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:48.638+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:49.022 INFO:journalctl@ceph.osd.3.smithi144.stdout:Apr 12 23:19:48 smithi144 podman[113328]: 2024-04-12 23:19:48.970195741 +0000 UTC m=+0.610770437 container died 905d6aaf84a237cc8b01f0a47aade2486986a2f4c496c6b8908303444fffa0a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T23:19:49.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:49.032+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:49.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:49.203+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:49.534 DEBUG:teuthology.orchestra.run.smithi144:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.3.service' 2024-04-12T23:19:49.576 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:49.576 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-12T23:19:49.576 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-12T23:19:49.576 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.4 2024-04-12T23:19:49.906 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:49.744+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:49.907 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:49 smithi181 systemd[1]: Stopping Ceph osd.4 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:49.907 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:49.652+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:49.907 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:49.919+0000 7f12a149f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:49.907 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:49.919+0000 7f12a149f700 -1 osd.4 159 *** Got signal Terminated *** 2024-04-12T23:19:49.907 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:49 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:49.919+0000 7f12a149f700 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:19:50.157 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:50 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:50.059+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:50.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:50 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:50.171+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:50.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:50 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:50.664+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:50.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:50 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:50.697+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:51.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:51 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:51.068+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:51.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:51 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:51.121+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:51.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:51 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:51.637+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:51.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:51 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:51.698+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:52.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:52 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:52.068+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:52.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:52 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:52.115+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:52.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:52 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:52.115+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:52.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:52 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:52.657+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:52.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:52 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:52.730+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:53.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:53.101+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:53.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:53.101+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:53.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:53.118+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:53.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:53.118+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:53.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:53.655+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:53.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:53.764+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:53.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:53 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:53.764+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:19:54.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:54.142+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:54.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:54.142+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:54.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:54.101+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:54.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:54.101+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:54.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:54.698+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:19.242676+0000 front 2024-04-12T23:19:19.242644+0000 (oldest deadline 2024-04-12T23:19:44.542508+0000) 2024-04-12T23:19:54.946 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4[57923]: 2024-04-12T23:19:54.698+0000 7f1299a8b700 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.044577+0000 front 2024-04-12T23:19:29.044496+0000 (oldest deadline 2024-04-12T23:19:53.744379+0000) 2024-04-12T23:19:54.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:54.766+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:54.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:54 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:54.766+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:19:55.420 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:55.101+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:55.420 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:55.101+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:55.420 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:55.115+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:55.420 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:55.115+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:55.697 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:55 smithi181 podman[78962]: 2024-04-12 23:19:55.433658063 +0000 UTC m=+5.641377708 container died d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb) 2024-04-12T23:19:56.081 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:55 smithi181 podman[78962]: 2024-04-12 23:19:55.886796519 +0000 UTC m=+6.094516212 container cleanup d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b277feb, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-12T23:19:56.081 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:55 smithi181 bash[78962]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4 2024-04-12T23:19:56.081 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:55.782+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:56.081 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:55 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:55.782+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:19:56.446 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:56 smithi181 podman[78981]: 2024-04-12 23:19:56.213465596 +0000 UTC m=+0.780231450 container remove d5b22101fa217a99c2ae625aa657483353ce7713c00f4a2409cfea4a946e6a16 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-12T23:19:56.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:56.094+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:56.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:56.094+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:56.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:56.145+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:56.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:56.145+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:56.759 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:56 smithi181 podman[79075]: 2024-04-12 23:19:56.406760404 +0000 UTC m=+0.018070689 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:19:56.760 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:56 smithi181 podman[79075]: 2024-04-12 23:19:56.547942077 +0000 UTC m=+0.159252343 container create fa349c05e7f80da4fa71959e253c15208d9f45831acba4e150c9b7e473fb9af6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-12T23:19:56.760 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:56 smithi181 podman[79075]: 2024-04-12 23:19:56.750142731 +0000 UTC m=+0.361452993 container init fa349c05e7f80da4fa71959e253c15208d9f45831acba4e150c9b7e473fb9af6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, RELEASE=squid-b277feb, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-12T23:19:56.760 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:56 smithi181 podman[79075]: 2024-04-12 23:19:56.753388708 +0000 UTC m=+0.364698974 container start fa349c05e7f80da4fa71959e253c15208d9f45831acba4e150c9b7e473fb9af6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, ceph=True, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409) 2024-04-12T23:19:57.032 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:56 smithi181 podman[79075]: 2024-04-12 23:19:56.892414816 +0000 UTC m=+0.503725095 container attach fa349c05e7f80da4fa71959e253c15208d9f45831acba4e150c9b7e473fb9af6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=) 2024-04-12T23:19:57.032 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:56.772+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:57.033 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:56 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:56.772+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:19:57.324 INFO:journalctl@ceph.osd.4.smithi181.stdout:Apr 12 23:19:57 smithi181 podman[79075]: 2024-04-12 23:19:57.046371344 +0000 UTC m=+0.657681626 container died fa349c05e7f80da4fa71959e253c15208d9f45831acba4e150c9b7e473fb9af6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb) 2024-04-12T23:19:57.324 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:57.051+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:57.325 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:57.051+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:57.325 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:57.152+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:57.325 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:57.152+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:57.628 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.4.service' 2024-04-12T23:19:57.669 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:19:57.669 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-12T23:19:57.669 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-12T23:19:57.670 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.5 2024-04-12T23:19:57.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:57 smithi181 systemd[1]: Stopping Ceph osd.5 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:19:57.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:57.797+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:57.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:57 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:57.797+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:19:58.446 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:58.048+0000 7fe0c8d97700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:19:58.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:58.048+0000 7fe0c8d97700 -1 osd.5 159 *** Got signal Terminated *** 2024-04-12T23:19:58.447 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:58.048+0000 7fe0c8d97700 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:19:58.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:58.100+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:58.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:58.100+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:58.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:58.135+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:58.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:58.135+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:19:59.083 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:58.754+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:19:59.083 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:58 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:58.754+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:19:59.084 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:59 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:59.096+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:19:59.084 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:19:59 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:19:59.096+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:19:59.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:59 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:59.105+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:19:59.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:19:59 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:19:59.105+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:00.043 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:59 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:59.794+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:20:00.043 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:19:59 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:19:59.794+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:20:00.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:00.143+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:00.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:00.143+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:00.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:00.056+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:00.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:00.056+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:00.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:00.056+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:01.031 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:00.745+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:20:01.032 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:00 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:00.745+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:20:01.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:01.103+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:01.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:01.103+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:01.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:01.103+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:01.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:01.044+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:01.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:01.044+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:01.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:01.044+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:02.049 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:01.766+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:20:02.050 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:01 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:01.766+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:20:02.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:02.121+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:02.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:02.121+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:02.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:02.121+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:02.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:02.062+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:02.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:02.062+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:02.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:02.062+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:03.077 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:02.815+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:21.873191+0000 front 2024-04-12T23:19:21.873178+0000 (oldest deadline 2024-04-12T23:19:45.373058+0000) 2024-04-12T23:20:03.078 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:02.815+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:29.374271+0000 front 2024-04-12T23:19:29.374387+0000 (oldest deadline 2024-04-12T23:19:53.474123+0000) 2024-04-12T23:20:03.078 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:02 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5[60183]: 2024-04-12T23:20:02.815+0000 7fe0c538b700 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.175077+0000 front 2024-04-12T23:19:38.174998+0000 (oldest deadline 2024-04-12T23:20:02.274914+0000) 2024-04-12T23:20:03.078 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:03.037+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:03.078 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:03.037+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:03.078 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:03.037+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:03.430 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:03.090+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:03.430 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:03.090+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:03.430 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:03 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:03.090+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:03.695 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:03 smithi181 podman[79194]: 2024-04-12 23:20:03.443241511 +0000 UTC m=+5.519601122 container died d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, RELEASE=squid-b277feb) 2024-04-12T23:20:03.946 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:03 smithi181 podman[79194]: 2024-04-12 23:20:03.807198001 +0000 UTC m=+5.883557612 container cleanup d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, RELEASE=squid-b277feb, io.buildah.version=1.33.5) 2024-04-12T23:20:03.947 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:03 smithi181 bash[79194]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5 2024-04-12T23:20:04.418 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79244]: 2024-04-12 23:20:04.091681576 +0000 UTC m=+0.648433136 container remove d4863d7c5c03e0b1fe5204a0fb747f134ca5e275a5b0b5e9d15b538299bbc448 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b277feb, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-12T23:20:04.418 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:04 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:04.123+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:04.418 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:04 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:04.123+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:04.418 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:04 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:04.123+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:04.418 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:04 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:04.053+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:04.419 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:04 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:04.053+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:04.419 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:04 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:04.053+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:04.697 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79338]: 2024-04-12 23:20:04.334661299 +0000 UTC m=+0.017910123 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:20:04.697 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79338]: 2024-04-12 23:20:04.449385465 +0000 UTC m=+0.132634321 container create ec8a2e30b7a4ee602c59fc0a3260d20fcdb3ea85bcba9f4dffe57ebaa13a6f80 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-12T23:20:04.697 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79338]: 2024-04-12 23:20:04.66988108 +0000 UTC m=+0.353129891 container init ec8a2e30b7a4ee602c59fc0a3260d20fcdb3ea85bcba9f4dffe57ebaa13a6f80 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:20:04.697 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79338]: 2024-04-12 23:20:04.673032096 +0000 UTC m=+0.356280908 container start ec8a2e30b7a4ee602c59fc0a3260d20fcdb3ea85bcba9f4dffe57ebaa13a6f80 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b277feb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:20:05.083 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79338]: 2024-04-12 23:20:04.761723067 +0000 UTC m=+0.444971904 container attach ec8a2e30b7a4ee602c59fc0a3260d20fcdb3ea85bcba9f4dffe57ebaa13a6f80 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409) 2024-04-12T23:20:05.083 INFO:journalctl@ceph.osd.5.smithi181.stdout:Apr 12 23:20:04 smithi181 podman[79338]: 2024-04-12 23:20:04.9652129 +0000 UTC m=+0.648461712 container died ec8a2e30b7a4ee602c59fc0a3260d20fcdb3ea85bcba9f4dffe57ebaa13a6f80 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-12T23:20:05.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:05.096+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:05.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:05.096+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:05.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:05.096+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:05.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:05.128+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:05.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:05.128+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:05.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:05.128+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:05.505 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.5.service' 2024-04-12T23:20:05.546 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:20:05.546 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-12T23:20:05.547 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-12T23:20:05.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.6 2024-04-12T23:20:05.871 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 systemd[1]: Stopping Ceph osd.6 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:20:06.196 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:06 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:06.074+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:06.196 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:06 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:06.074+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:06.196 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:06 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:06.074+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:06.197 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:05.884+0000 7f09a3e8e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:20:06.197 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:05.884+0000 7f09a3e8e700 -1 osd.6 159 *** Got signal Terminated *** 2024-04-12T23:20:06.197 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:05 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:05.884+0000 7f09a3e8e700 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:20:06.197 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:06 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:06.111+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:06.197 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:06 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:06.111+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:06.197 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:06 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:06.111+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:07.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:07 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:07.137+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:07.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:07 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:07.137+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:07.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:07 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:07.137+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:07.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:07 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:07.051+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:07.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:07 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:07.051+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:07.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:07 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:07.051+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:08.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:08.089+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:08.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:08.089+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:08.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:08.089+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:08.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:08.110+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:08.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:08.110+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:08.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:08.110+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:08.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:08 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:08.110+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.900181+0000 front 2024-04-12T23:19:44.900143+0000 (oldest deadline 2024-04-12T23:20:07.800066+0000) 2024-04-12T23:20:09.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:09.084+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:09.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:09.084+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:09.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:09.084+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:09.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:09.084+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.900181+0000 front 2024-04-12T23:19:44.900143+0000 (oldest deadline 2024-04-12T23:20:07.800066+0000) 2024-04-12T23:20:09.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:09.086+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:09.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:09.086+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:09.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:09 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:09.086+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:10.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:10.074+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:10.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:10.074+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:10.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:10.074+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:10.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:10.132+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:22.297142+0000 front 2024-04-12T23:19:22.297088+0000 (oldest deadline 2024-04-12T23:19:46.996844+0000) 2024-04-12T23:20:10.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:10.132+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:28.697813+0000 front 2024-04-12T23:19:28.697855+0000 (oldest deadline 2024-04-12T23:19:52.797701+0000) 2024-04-12T23:20:10.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:10.132+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:38.499376+0000 front 2024-04-12T23:19:38.499314+0000 (oldest deadline 2024-04-12T23:20:00.799306+0000) 2024-04-12T23:20:10.447 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:10 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6[62452]: 2024-04-12T23:20:10.132+0000 7f099c47a700 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.900181+0000 front 2024-04-12T23:19:44.900143+0000 (oldest deadline 2024-04-12T23:20:07.800066+0000) 2024-04-12T23:20:11.446 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:11 smithi181 podman[79445]: 2024-04-12 23:20:11.303557616 +0000 UTC m=+5.520256326 container died 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-12T23:20:11.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:11 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:11.078+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:11.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:11 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:11.078+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:11.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:11 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:11.078+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:11.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:11 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:11.078+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:11.946 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:11 smithi181 podman[79445]: 2024-04-12 23:20:11.659104192 +0000 UTC m=+5.875802910 container cleanup 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-12T23:20:11.946 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:11 smithi181 bash[79445]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6 2024-04-12T23:20:12.238 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:11 smithi181 podman[79465]: 2024-04-12 23:20:11.960397839 +0000 UTC m=+0.661146312 container remove 6c1178b4b3f37e19be20b882f8bf9ccdcc1ea658d9affed33fbeb3b9a6877ce6 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T23:20:12.239 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:12 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:12.071+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:12.239 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:12 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:12.071+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:12.239 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:12 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:12.071+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:12.239 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:12 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:12.071+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:12.525 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:12 smithi181 podman[79555]: 2024-04-12 23:20:12.155307603 +0000 UTC m=+0.018572832 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:20:12.526 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:12 smithi181 podman[79555]: 2024-04-12 23:20:12.252797725 +0000 UTC m=+0.116062952 container create 21faa8b90ba01c60b53500daa6cb682aef7fada092dedc7de2b1c0d938407680 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , RELEASE=squid-b277feb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-12T23:20:12.526 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:12 smithi181 podman[79555]: 2024-04-12 23:20:12.463809268 +0000 UTC m=+0.327074486 container init 21faa8b90ba01c60b53500daa6cb682aef7fada092dedc7de2b1c0d938407680 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240409, io.buildah.version=1.33.5, RELEASE=squid-b277feb) 2024-04-12T23:20:12.526 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:12 smithi181 podman[79555]: 2024-04-12 23:20:12.466865785 +0000 UTC m=+0.330131007 container start 21faa8b90ba01c60b53500daa6cb682aef7fada092dedc7de2b1c0d938407680 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=) 2024-04-12T23:20:12.526 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:12 smithi181 podman[79555]: 2024-04-12 23:20:12.538707415 +0000 UTC m=+0.401972652 container attach 21faa8b90ba01c60b53500daa6cb682aef7fada092dedc7de2b1c0d938407680 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-12T23:20:12.946 INFO:journalctl@ceph.osd.6.smithi181.stdout:Apr 12 23:20:12 smithi181 podman[79555]: 2024-04-12 23:20:12.760141062 +0000 UTC m=+0.623406287 container died 21faa8b90ba01c60b53500daa6cb682aef7fada092dedc7de2b1c0d938407680 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-b277feb, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-12T23:20:13.303 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.043+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:13.303 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.043+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:13.303 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.043+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:13.303 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.043+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:13.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.6.service' 2024-04-12T23:20:13.367 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:20:13.367 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-12T23:20:13.368 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-12T23:20:13.368 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.7 2024-04-12T23:20:13.696 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 systemd[1]: Stopping Ceph osd.7 for 9965523c-f91f-11ee-bc8d-c7b262605968... 2024-04-12T23:20:14.014 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.719+0000 7fe5a80d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-12T23:20:14.014 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.719+0000 7fe5a80d0700 -1 osd.7 159 *** Got signal Terminated *** 2024-04-12T23:20:14.014 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:13 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:13.719+0000 7fe5a80d0700 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-12T23:20:14.017 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:14 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:14.027+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:14.017 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:14 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:14.027+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:14.017 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:14 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:14.027+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:14.017 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:14 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:14.027+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:15.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:15 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:15.075+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:15.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:15 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:15.075+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:15.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:15 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:15.075+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:15.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:15 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:15.075+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:16.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:16.076+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:16.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:16.076+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:16.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:16.076+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:16.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:16.076+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:16.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:16 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:16.076+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-04-12T23:19:50.522198+0000 front 2024-04-12T23:19:50.522218+0000 (oldest deadline 2024-04-12T23:20:15.822261+0000) 2024-04-12T23:20:17.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:17 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:17.055+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:17.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:17 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:17.055+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:17.446 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:17 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:17.055+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:17.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:17 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:17.055+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:17.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:17 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:17.055+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-04-12T23:19:50.522198+0000 front 2024-04-12T23:19:50.522218+0000 (oldest deadline 2024-04-12T23:20:15.822261+0000) 2024-04-12T23:20:18.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:18 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:18.010+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6806 osd.0 since back 2024-04-12T23:19:20.619100+0000 front 2024-04-12T23:19:20.619179+0000 (oldest deadline 2024-04-12T23:19:46.519009+0000) 2024-04-12T23:20:18.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:18 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:18.010+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6814 osd.1 since back 2024-04-12T23:19:26.519526+0000 front 2024-04-12T23:19:26.519592+0000 (oldest deadline 2024-04-12T23:19:51.819674+0000) 2024-04-12T23:20:18.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:18 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:18.010+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6822 osd.2 since back 2024-04-12T23:19:35.320785+0000 front 2024-04-12T23:19:35.320722+0000 (oldest deadline 2024-04-12T23:20:00.020610+0000) 2024-04-12T23:20:18.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:18 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:18.010+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.144:6830 osd.3 since back 2024-04-12T23:19:44.621874+0000 front 2024-04-12T23:19:44.621968+0000 (oldest deadline 2024-04-12T23:20:10.521813+0000) 2024-04-12T23:20:18.447 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:18 smithi181 ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7[64721]: 2024-04-12T23:20:18.010+0000 7fe5a46c4700 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.181:6804 osd.4 since back 2024-04-12T23:19:50.522198+0000 front 2024-04-12T23:19:50.522218+0000 (oldest deadline 2024-04-12T23:20:15.822261+0000) 2024-04-12T23:20:19.409 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:19 smithi181 podman[79658]: 2024-04-12 23:20:19.147117962 +0000 UTC m=+5.548428991 container died 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240409, ceph=True) 2024-04-12T23:20:19.696 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:19 smithi181 podman[79658]: 2024-04-12 23:20:19.502755094 +0000 UTC m=+5.904066154 container cleanup 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=squid-b277feb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240409, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-12T23:20:19.696 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:19 smithi181 bash[79658]: ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7 2024-04-12T23:20:20.099 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:19 smithi181 podman[79720]: 2024-04-12 23:20:19.762268604 +0000 UTC m=+0.613833161 container remove 54cb91ee3688025160f7aa5f0d220cb6622dab214ceb8071c02348c6c6535200 (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b277feb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240409, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-12T23:20:20.360 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:20 smithi181 podman[79809]: 2024-04-12 23:20:20.016161732 +0000 UTC m=+0.017485138 image pull 668895944f0ec64b67a5b7564961d2756f2966e5d4b9338516300dbec9984951 quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec 2024-04-12T23:20:20.360 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:20 smithi181 podman[79809]: 2024-04-12 23:20:20.137998229 +0000 UTC m=+0.139321603 container create 452f564ca7799678938a70567bc54639d13754cf77652212d01269b68d3be52a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240409, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, ceph=True, org.label-schema.schema-version=1.0) 2024-04-12T23:20:20.360 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:20 smithi181 podman[79809]: 2024-04-12 23:20:20.374114105 +0000 UTC m=+0.375437492 container init 452f564ca7799678938a70567bc54639d13754cf77652212d01269b68d3be52a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240409, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb) 2024-04-12T23:20:20.656 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:20 smithi181 podman[79809]: 2024-04-12 23:20:20.377375644 +0000 UTC m=+0.378699021 container start 452f564ca7799678938a70567bc54639d13754cf77652212d01269b68d3be52a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240409) 2024-04-12T23:20:20.656 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:20 smithi181 podman[79809]: 2024-04-12 23:20:20.473618036 +0000 UTC m=+0.474941426 container attach 452f564ca7799678938a70567bc54639d13754cf77652212d01269b68d3be52a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-b277feb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240409) 2024-04-12T23:20:20.945 INFO:journalctl@ceph.osd.7.smithi181.stdout:Apr 12 23:20:20 smithi181 podman[79809]: 2024-04-12 23:20:20.670039793 +0000 UTC m=+0.671363172 container died 452f564ca7799678938a70567bc54639d13754cf77652212d01269b68d3be52a (image=quay.ceph.io/ceph-ci/ceph@sha256:7a231d9b74270e9fc5dba084646cc7a148bd8e84d7c66434198a233cfee0f4ec, name=ceph-9965523c-f91f-11ee-bc8d-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b277feb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240409, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-12T23:20:21.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9965523c-f91f-11ee-bc8d-c7b262605968@osd.7.service' 2024-04-12T23:20:21.387 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-12T23:20:21.387 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-12T23:20:21.388 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 --force --keep-logs 2024-04-12T23:23:19.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 --force --keep-logs 2024-04-12T23:23:40.438 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T23:23:40.476 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-12T23:23:40.506 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-12T23:23:40.508 DEBUG:teuthology.misc:Transferring archived files from smithi144:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/remote/smithi144/crash 2024-04-12T23:23:40.509 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/crash -- . 2024-04-12T23:23:40.563 INFO:teuthology.orchestra.run.smithi144.stderr:tar: /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T23:23:40.564 INFO:teuthology.orchestra.run.smithi144.stderr:tar: Error is not recoverable: exiting now 2024-04-12T23:23:40.565 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/remote/smithi181/crash 2024-04-12T23:23:40.566 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/crash -- . 2024-04-12T23:23:40.607 INFO:teuthology.orchestra.run.smithi181.stderr:tar: /var/lib/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/crash: Cannot open: No such file or directory 2024-04-12T23:23:40.607 INFO:teuthology.orchestra.run.smithi181.stderr:tar: Error is not recoverable: exiting now 2024-04-12T23:23:40.608 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-12T23:23:40.608 DEBUG:teuthology.orchestra.run.smithi144:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T23:23:40.641 INFO:teuthology.orchestra.run.smithi144.stdout:2024-04-12T22:59:59.987381+0000 mon.smithi144 (mon.0) 504 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T23:23:40.641 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-12T23:23:40.641 DEBUG:teuthology.orchestra.run.smithi144:> sudo egrep '\[SEC\]' /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T23:23:40.712 DEBUG:teuthology.orchestra.run.smithi144:> sudo egrep '\[ERR\]' /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T23:23:40.784 DEBUG:teuthology.orchestra.run.smithi144:> sudo egrep '\[WRN\]' /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-12T23:23:40.854 INFO:teuthology.orchestra.run.smithi144.stdout:2024-04-12T22:59:59.987381+0000 mon.smithi144 (mon.0) 504 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-12T23:23:40.854 INFO:tasks.cephadm:Compressing logs... 2024-04-12T23:23:40.854 DEBUG:teuthology.orchestra.run.smithi144:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T23:23:40.896 DEBUG:teuthology.orchestra.run.smithi181:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-12T23:23:40.923 INFO:teuthology.orchestra.run.smithi144.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose: No such file or directory -- 2024-04-12T23:23:40.924 INFO:teuthology.orchestra.run.smithi144.stderr: /var/log/ceph/cephadm.log 2024-04-12T23:23:40.924 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.audit.log 2024-04-12T23:23:40.924 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.1.log 2024-04-12T23:23:40.924 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi144.alilcy.log 2024-04-12T23:23:40.928 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mgr.smithi144.rlueky.log 2024-04-12T23:23:40.928 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.3.log 2024-04-12T23:23:40.928 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi144.alilcy.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mgr.smithi144.rlueky.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.0.log 2024-04-12T23:23:40.928 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.3.log: 89.0% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi144.alilcy.log.gzgzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mon.smithi144.log 2024-04-12T23:23:40.928 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T23:23:40.928 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.cephadm.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:: No such file or directory 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.6.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mon.smithi181.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose --/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.6.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.audit.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mon.smithi181.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi181.gciyns.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.4.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.audit.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi181.gciyns.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.7.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.5.log 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr: 87.7% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mds.foofs.smithi181.gciyns.log.gz 2024-04-12T23:23:40.930 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.cephadm.log 2024-04-12T23:23:40.931 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log 2024-04-12T23:23:40.931 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.5.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.cephadm.log.gz 2024-04-12T23:23:40.931 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mgr.smithi181.whsrmz.log 2024-04-12T23:23:40.931 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log: 91.6% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.audit.log.gz 2024-04-12T23:23:40.932 INFO:teuthology.orchestra.run.smithi181.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T23:23:40.932 INFO:teuthology.orchestra.run.smithi181.stderr: 87.0% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log.gz 2024-04-12T23:23:40.932 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-volume.log 2024-04-12T23:23:40.933 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.0.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mon.smithi144.log: 91.5% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.audit.log.gz 2024-04-12T23:23:40.933 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log 2024-04-12T23:23:40.933 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.cephadm.log.gz 2024-04-12T23:23:40.933 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.2.log 2024-04-12T23:23:40.936 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log: 87.0% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph.log.gz 2024-04-12T23:23:40.936 INFO:teuthology.orchestra.run.smithi144.stderr:gzip -5 --verbose -- /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-volume.log 2024-04-12T23:23:40.948 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mgr.smithi181.whsrmz.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-volume.log: 91.5% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-volume.log.gz 2024-04-12T23:23:40.961 INFO:teuthology.orchestra.run.smithi144.stderr:/var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.2.log: /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-volume.log: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-12T23:23:40.967 INFO:teuthology.orchestra.run.smithi144.stderr: 91.4% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-volume.log.gz 2024-04-12T23:23:41.004 INFO:teuthology.orchestra.run.smithi181.stderr: 89.6% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mgr.smithi181.whsrmz.log.gz 2024-04-12T23:23:41.324 INFO:teuthology.orchestra.run.smithi144.stderr: 88.5% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mgr.smithi144.rlueky.log.gz 2024-04-12T23:23:41.422 INFO:teuthology.orchestra.run.smithi181.stderr: 92.6% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mon.smithi181.log.gz 2024-04-12T23:23:42.945 INFO:teuthology.orchestra.run.smithi144.stderr: 92.0% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-mon.smithi144.log.gz 2024-04-12T23:23:53.979 INFO:teuthology.orchestra.run.smithi144.stderr: 94.5% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.2.log.gz 2024-04-12T23:23:56.758 INFO:teuthology.orchestra.run.smithi181.stderr: 93.9% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.4.log.gz 2024-04-12T23:23:58.663 INFO:teuthology.orchestra.run.smithi181.stderr: 94.2% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.6.log.gz 2024-04-12T23:23:59.455 INFO:teuthology.orchestra.run.smithi144.stderr: 94.1% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.0.log.gz 2024-04-12T23:23:59.578 INFO:teuthology.orchestra.run.smithi144.stderr: 94.2% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.1.log.gz 2024-04-12T23:24:01.720 INFO:teuthology.orchestra.run.smithi144.stderr: 94.1% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.3.log.gz 2024-04-12T23:24:01.722 INFO:teuthology.orchestra.run.smithi144.stderr: 2024-04-12T23:24:01.722 INFO:teuthology.orchestra.run.smithi144.stderr:real 0m20.813s 2024-04-12T23:24:01.722 INFO:teuthology.orchestra.run.smithi144.stderr:user 1m10.592s 2024-04-12T23:24:01.722 INFO:teuthology.orchestra.run.smithi144.stderr:sys 0m2.806s 2024-04-12T23:24:02.028 INFO:teuthology.orchestra.run.smithi181.stderr: 94.2% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.5.log.gz 2024-04-12T23:24:06.072 INFO:teuthology.orchestra.run.smithi181.stderr: 94.1% -- replaced with /var/log/ceph/9965523c-f91f-11ee-bc8d-c7b262605968/ceph-osd.7.log.gz 2024-04-12T23:24:06.074 INFO:teuthology.orchestra.run.smithi181.stderr: 2024-04-12T23:24:06.074 INFO:teuthology.orchestra.run.smithi181.stderr:real 0m25.162s 2024-04-12T23:24:06.074 INFO:teuthology.orchestra.run.smithi181.stderr:user 1m17.097s 2024-04-12T23:24:06.074 INFO:teuthology.orchestra.run.smithi181.stderr:sys 0m3.088s 2024-04-12T23:24:06.074 INFO:tasks.cephadm:Archiving logs... 2024-04-12T23:24:06.075 DEBUG:teuthology.misc:Transferring archived files from smithi144:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/remote/smithi144/log 2024-04-12T23:24:06.076 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T23:24:15.736 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/remote/smithi181/log 2024-04-12T23:24:15.738 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-12T23:24:26.697 INFO:tasks.cephadm:Removing cluster... 2024-04-12T23:24:26.698 DEBUG:teuthology.orchestra.run.smithi144:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 --force 2024-04-12T23:24:27.075 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9965523c-f91f-11ee-bc8d-c7b262605968 --force 2024-04-12T23:24:27.463 INFO:tasks.cephadm:Removing cephadm ... 2024-04-12T23:24:27.464 DEBUG:teuthology.orchestra.run.smithi144:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T23:24:27.479 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-12T23:24:27.493 INFO:tasks.cephadm:Teardown complete 2024-04-12T23:24:27.494 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-12T23:24:27.506 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-12T23:24:27.506 DEBUG:teuthology.orchestra.run.smithi144:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T23:24:27.521 DEBUG:teuthology.orchestra.run.smithi181:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-12T23:24:27.532 INFO:teuthology.orchestra.run.smithi144.stderr:bash: line 1: ntpq: command not found 2024-04-12T23:24:27.537 INFO:teuthology.orchestra.run.smithi144.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T23:24:27.537 INFO:teuthology.orchestra.run.smithi144.stdout:=============================================================================== 2024-04-12T23:24:27.537 INFO:teuthology.orchestra.run.smithi144.stdout:^* hv01.front.sepia.ceph.com 3 6 377 56 +700us[ +828us] +/- 38ms 2024-04-12T23:24:27.538 INFO:teuthology.orchestra.run.smithi144.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 55 -902us[ -902us] +/- 60ms 2024-04-12T23:24:27.538 INFO:teuthology.orchestra.run.smithi144.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 +12us[ +12us] +/- 71ms 2024-04-12T23:24:27.538 INFO:teuthology.orchestra.run.smithi144.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T23:24:27.547 INFO:teuthology.orchestra.run.smithi181.stderr:bash: line 1: ntpq: command not found 2024-04-12T23:24:27.551 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-12T23:24:27.551 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2024-04-12T23:24:27.552 INFO:teuthology.orchestra.run.smithi181.stdout:^* hv01.front.sepia.ceph.com 3 6 377 54 +671us[ +758us] +/- 38ms 2024-04-12T23:24:27.552 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 54 -918us[ -918us] +/- 60ms 2024-04-12T23:24:27.552 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 -11us[ +76us] +/- 71ms 2024-04-12T23:24:27.552 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-12T23:24:27.552 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-12T23:24:27.563 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-12T23:24:27.564 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-12T23:24:27.587 DEBUG:teuthology.orchestra.run.smithi144:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T23:24:27.671 DEBUG:teuthology.orchestra.run.smithi181:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-12T23:24:27.748 DEBUG:teuthology.orchestra.run.smithi144:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T23:24:27.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T23:24:27.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-12T23:24:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T23:24:27.808 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-12T23:24:27.819 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-12T23:24:27.837 INFO:teuthology.task.internal:Duration was 2065.514840 seconds 2024-04-12T23:24:27.837 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-12T23:24:27.872 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-12T23:24:27.872 DEBUG:teuthology.orchestra.run.smithi144:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T23:24:27.874 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-12T23:24:27.915 INFO:teuthology.orchestra.run.smithi144.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T23:24:27.917 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-12T23:24:28.319 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-12T23:24:28.319 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi144.front.sepia.ceph.com 2024-04-12T23:24:28.319 DEBUG:teuthology.orchestra.run.smithi144:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T23:24:28.379 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi181.front.sepia.ceph.com 2024-04-12T23:24:28.379 DEBUG:teuthology.orchestra.run.smithi181:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-12T23:24:28.408 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-12T23:24:28.409 DEBUG:teuthology.orchestra.run.smithi144:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T23:24:28.422 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-12T23:24:28.477 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-12T23:24:28.477 DEBUG:teuthology.orchestra.run.smithi144:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T23:24:28.491 DEBUG:teuthology.orchestra.run.smithi181:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-12T23:24:42.272 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-12T23:24:42.284 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-12T23:24:42.285 DEBUG:teuthology.orchestra.run.smithi144:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T23:24:42.312 DEBUG:teuthology.orchestra.run.smithi181:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-12T23:24:42.344 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-12T23:24:42.355 DEBUG:teuthology.orchestra.run.smithi144:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T23:24:42.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-12T23:24:42.380 INFO:teuthology.orchestra.run.smithi144.stdout:kernel.core_pattern = core 2024-04-12T23:24:42.412 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = core 2024-04-12T23:24:42.439 DEBUG:teuthology.orchestra.run.smithi144:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T23:24:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T23:24:42.461 DEBUG:teuthology.orchestra.run.smithi181:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-12T23:24:42.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-12T23:24:42.494 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-12T23:24:42.505 INFO:teuthology.task.internal:Transferring archived files... 2024-04-12T23:24:42.506 DEBUG:teuthology.misc:Transferring archived files from smithi144:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/remote/smithi144 2024-04-12T23:24:42.507 DEBUG:teuthology.orchestra.run.smithi144:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T23:24:42.569 DEBUG:teuthology.misc:Transferring archived files from smithi181:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631/remote/smithi181 2024-04-12T23:24:42.569 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-12T23:24:42.612 INFO:teuthology.task.internal:Removing archive directory... 2024-04-12T23:24:42.613 DEBUG:teuthology.orchestra.run.smithi144:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T23:24:42.615 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-12T23:24:42.661 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-12T23:24:42.674 INFO:teuthology.task.internal:Not uploading archives. 2024-04-12T23:24:42.674 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-12T23:24:42.728 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-12T23:24:42.729 DEBUG:teuthology.orchestra.run.smithi144:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T23:24:42.731 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-12T23:24:42.743 INFO:teuthology.orchestra.run.smithi144.stdout: 265307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 23:24 /home/ubuntu/cephtest 2024-04-12T23:24:42.746 INFO:teuthology.orchestra.run.smithi181.stdout: 265764 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 12 23:24 /home/ubuntu/cephtest 2024-04-12T23:24:42.747 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-12T23:24:42.774 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-12T23:24:42.842 INFO:teuthology.nuke:Checking targets against current locks 2024-04-12T23:24:42.870 DEBUG:teuthology.nuke:shortname: smithi144 2024-04-12T23:24:42.871 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T23:24:42.891 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi144.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 22:41:50.914611', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNdaf8Xngm/oFVwUNyqQnojKjbkKNh5z28+XuesW+Jyzli78mid3XcT9Rj+Qdtvane9+STVG0INvO5lthV1T0yw='} 2024-04-12T23:24:42.896 DEBUG:teuthology.nuke:shortname: smithi181 2024-04-12T23:24:42.896 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-12T23:24:42.917 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-12_21:08:02-orch-squid-distro-default-smithi/7654631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-12 22:41:50.915565', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEgMIC+qb4bkv4MhnV+yPCvTI1sRLxyEXDczVoeRf4gCPRqTFLnpOZ5YNIxrugSU1dg7YKqG7kxBl877Suu7z/I='} 2024-04-12T23:24:42.944 INFO:teuthology.orchestra.console.smithi144:Power off 2024-04-12T23:24:42.944 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T23:24:42.962 INFO:teuthology.orchestra.console.smithi181:Power off 2024-04-12T23:24:42.962 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-12T23:24:42.972 DEBUG:teuthology.orchestra.console.smithi144:power off output: Chassis Power Control: Down/Off 2024-04-12T23:24:42.972 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:42.987 DEBUG:teuthology.orchestra.console.smithi181:power off output: Chassis Power Control: Down/Off 2024-04-12T23:24:42.987 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:42.998 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is on 2024-04-12T23:24:43.013 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2024-04-12T23:24:47.000 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:47.014 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:47.110 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is on 2024-04-12T23:24:47.124 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2024-04-12T23:24:51.112 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:51.125 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:51.223 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is on 2024-04-12T23:24:51.237 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is off 2024-04-12T23:24:51.338 INFO:teuthology.orchestra.console.smithi181:Power off completed 2024-04-12T23:24:55.227 DEBUG:teuthology.orchestra.console.smithi144:pexpect command: ipmitool -H smithi144.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-12T23:24:55.337 DEBUG:teuthology.orchestra.console.smithi144:check power output: Chassis Power is off 2024-04-12T23:24:55.438 INFO:teuthology.orchestra.console.smithi144:Power off completed 2024-04-12T23:24:55.542 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2065.514839887619 failure_reason: '"2024-04-12T22:59:59.987381+0000 mon.smithi144 (mon.0) 504 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log' owner: scheduled_teuthology@teuthology success: false 2024-04-12T23:24:55.542 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-12T23:24:55.643 INFO:teuthology.run:FAIL